{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mERROR: Could not find a version that satisfies the requirement torch==2.2 (from versions: 1.0.0, 1.0.1, 1.0.1.post2, 1.1.0, 1.1.0.post2, 1.2.0, 1.3.0, 1.3.0.post2, 1.3.1, 1.4.0, 1.5.0, 1.5.1, 1.6.0, 1.7.0, 1.7.1, 1.8.0, 1.8.1, 1.9.0, 1.9.1, 1.10.0, 1.10.1, 1.10.2, 1.11.0, 1.12.0, 1.12.1, 1.13.0, 1.13.1)\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31mERROR: No matching distribution found for torch==2.2\u001b[0m\u001b[31m\n",
      "\u001b[0mRequirement already satisfied: matplotlib==3.5.2 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (3.5.2)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from matplotlib==3.5.2) (1.4.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from matplotlib==3.5.2) (23.2)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from matplotlib==3.5.2) (2.8.2)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from matplotlib==3.5.2) (3.1.1)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from matplotlib==3.5.2) (4.38.0)\n",
      "Requirement already satisfied: numpy>=1.17 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from matplotlib==3.5.2) (1.21.6)\n",
      "Requirement already satisfied: cycler>=0.10 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from matplotlib==3.5.2) (0.11.0)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from matplotlib==3.5.2) (9.3.0)\n",
      "Requirement already satisfied: typing-extensions in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from kiwisolver>=1.0.1->matplotlib==3.5.2) (4.7.1)\n",
      "Requirement already satisfied: six>=1.5 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from python-dateutil>=2.7->matplotlib==3.5.2) (1.16.0)\n",
      "Requirement already satisfied: scikit-image==0.19.3 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (0.19.3)\n",
      "Requirement already satisfied: tifffile>=2019.7.26 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from scikit-image==0.19.3) (2021.11.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from scikit-image==0.19.3) (23.2)\n",
      "Requirement already satisfied: imageio>=2.4.1 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from scikit-image==0.19.3) (2.31.2)\n",
      "Requirement already satisfied: scipy>=1.4.1 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from scikit-image==0.19.3) (1.7.3)\n",
      "Requirement already satisfied: networkx>=2.2 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from scikit-image==0.19.3) (2.6.3)\n",
      "Requirement already satisfied: numpy>=1.17.0 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from scikit-image==0.19.3) (1.21.6)\n",
      "Requirement already satisfied: pillow!=7.1.0,!=7.1.1,!=8.3.0,>=6.1.0 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from scikit-image==0.19.3) (9.3.0)\n",
      "Requirement already satisfied: PyWavelets>=1.1.1 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from scikit-image==0.19.3) (1.3.0)\n",
      "Requirement already satisfied: atari-py==0.2.9 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (0.2.9)\n",
      "Requirement already satisfied: six in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from atari-py==0.2.9) (1.16.0)\n",
      "Requirement already satisfied: numpy in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from atari-py==0.2.9) (1.21.6)\n",
      "Requirement already satisfied: opencv-python==4.6.0.66 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (4.6.0.66)\n",
      "Requirement already satisfied: numpy>=1.14.5 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from opencv-python==4.6.0.66) (1.21.6)\n",
      "Found existing installation: gym 0.24.0\n",
      "Uninstalling gym-0.24.0:\n",
      "  Successfully uninstalled gym-0.24.0\n",
      "Collecting gym[accept-rom-license,atari]==0.24.0\n",
      "  Using cached gym-0.24.0-py3-none-any.whl\n",
      "Requirement already satisfied: gym-notices>=0.0.4 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from gym[accept-rom-license,atari]==0.24.0) (0.0.8)\n",
      "Requirement already satisfied: numpy>=1.18.0 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from gym[accept-rom-license,atari]==0.24.0) (1.21.6)\n",
      "Requirement already satisfied: cloudpickle>=1.2.0 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from gym[accept-rom-license,atari]==0.24.0) (1.6.0)\n",
      "Requirement already satisfied: importlib-metadata>=4.8.0 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from gym[accept-rom-license,atari]==0.24.0) (4.13.0)\n",
      "Requirement already satisfied: autorom[accept-rom-license]~=0.4.2 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from gym[accept-rom-license,atari]==0.24.0) (0.4.2)\n",
      "Collecting ale-py~=0.7.5\n",
      "  Using cached ale_py-0.7.5-cp37-cp37m-macosx_10_15_x86_64.whl (1.1 MB)\n",
      "Requirement already satisfied: importlib-resources in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from ale-py~=0.7.5->gym[accept-rom-license,atari]==0.24.0) (5.12.0)\n",
      "Requirement already satisfied: tqdm in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from autorom[accept-rom-license]~=0.4.2->gym[accept-rom-license,atari]==0.24.0) (4.66.2)\n",
      "Requirement already satisfied: click in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from autorom[accept-rom-license]~=0.4.2->gym[accept-rom-license,atari]==0.24.0) (8.1.7)\n",
      "Requirement already satisfied: requests in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from autorom[accept-rom-license]~=0.4.2->gym[accept-rom-license,atari]==0.24.0) (2.31.0)\n",
      "Requirement already satisfied: AutoROM.accept-rom-license in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from autorom[accept-rom-license]~=0.4.2->gym[accept-rom-license,atari]==0.24.0) (0.6.1)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from importlib-metadata>=4.8.0->gym[accept-rom-license,atari]==0.24.0) (4.7.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from importlib-metadata>=4.8.0->gym[accept-rom-license,atari]==0.24.0) (3.15.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from requests->autorom[accept-rom-license]~=0.4.2->gym[accept-rom-license,atari]==0.24.0) (3.3.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from requests->autorom[accept-rom-license]~=0.4.2->gym[accept-rom-license,atari]==0.24.0) (2022.12.7)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from requests->autorom[accept-rom-license]~=0.4.2->gym[accept-rom-license,atari]==0.24.0) (3.6)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from requests->autorom[accept-rom-license]~=0.4.2->gym[accept-rom-license,atari]==0.24.0) (2.0.7)\n",
      "Installing collected packages: gym, ale-py\n",
      "  Attempting uninstall: ale-py\n",
      "    Found existing installation: ale-py 0.7.3\n",
      "    Uninstalling ale-py-0.7.3:\n",
      "      Successfully uninstalled ale-py-0.7.3\n",
      "Successfully installed ale-py-0.7.5 gym-0.24.0\n",
      "Collecting ale-py==0.7.3\n",
      "  Using cached ale_py-0.7.3-cp37-cp37m-macosx_10_15_x86_64.whl (1.1 MB)\n",
      "Requirement already satisfied: importlib-resources in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from ale-py==0.7.3) (5.12.0)\n",
      "Requirement already satisfied: importlib-metadata in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from ale-py==0.7.3) (4.13.0)\n",
      "Requirement already satisfied: numpy in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from ale-py==0.7.3) (1.21.6)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from importlib-metadata->ale-py==0.7.3) (4.7.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from importlib-metadata->ale-py==0.7.3) (3.15.0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Installing collected packages: ale-py\n",
      "  Attempting uninstall: ale-py\n",
      "    Found existing installation: ale-py 0.7.5\n",
      "    Uninstalling ale-py-0.7.5:\n",
      "      Successfully uninstalled ale-py-0.7.5\n",
      "Successfully installed ale-py-0.7.3\n",
      "Requirement already satisfied: importlib-metadata==4.13.0 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (4.13.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from importlib-metadata==4.13.0) (3.15.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4 in /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages (from importlib-metadata==4.13.0) (4.7.1)\n"
     ]
    }
   ],
   "source": [
    "# requires python3.7\n",
    "!pip install torch==2.2\n",
    "!pip install matplotlib==3.5.2\n",
    "!pip install scikit-image==0.19.3\n",
    "!pip install atari-py==0.2.9\n",
    "!pip install opencv-python==4.6.0.66\n",
    "!pip uninstall -y 'gym[atari,accept-rom-license]'\n",
    "!pip install 'gym[atari,accept-rom-license]==0.24.0'\n",
    "!pip install ale-py==0.7.3\n",
    "!pip install importlib-metadata==4.13.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-02-19 15:36:15--  http://www.atarimania.com/roms/Roms.rar\n",
      "Resolving www.atarimania.com (www.atarimania.com)... 195.154.81.199\n",
      "Connecting to www.atarimania.com (www.atarimania.com)|195.154.81.199|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 19612325 (19M) [application/x-rar-compressed]\n",
      "Saving to: ‘Roms.rar.3’\n",
      "\n",
      "Roms.rar.3          100%[===================>]  18.70M  4.64MB/s    in 4.1s    \n",
      "\n",
      "2024-02-19 15:36:19 (4.51 MB/s) - ‘Roms.rar.3’ saved [19612325/19612325]\n",
      "\n",
      "--2024-02-19 15:36:19--  http://./\n",
      "Resolving . (.)... failed: nodename nor servname provided, or not known.\n",
      "wget: unable to resolve host address ‘.’\n",
      "FINISHED --2024-02-19 15:36:19--\n",
      "Total wall clock time: 4.3s\n",
      "Downloaded: 1 files, 19M in 4.1s (4.51 MB/s)\n"
     ]
    }
   ],
   "source": [
    "!wget http://www.atarimania.com/roms/Roms.rar ./"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m==>\u001b[0m \u001b[1mDownloading https://formulae.brew.sh/api/formula.jws.json\u001b[0m\n",
      "\n",
      "\u001b[34m==>\u001b[0m \u001b[1mDownloading https://formulae.brew.sh/api/cask.jws.json\u001b[0m\n",
      "\n",
      "\u001b[33mWarning:\u001b[0m unar 1.10.8 is already installed and up-to-date.\n",
      "To reinstall 1.10.8, run:\n",
      "  brew reinstall unar\n",
      "./Roms.rar: RAR\n",
      "\"Roms\" already exists.\n",
      "(r)ename to \"Roms-1\", (R)ename all, (o)verwrite, (O)verwrite all, (s)kip, (S)kip all, (q)uit? ^C\n"
     ]
    }
   ],
   "source": [
    "!brew install unar\n",
    "!unar ./Roms.rar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "copying koolaid.bin from ./Roms/ROMS/Kool-Aid Man (Kool Aid Pitcher Man) (1983) (M Network, Stephen Tatsumi, Jane Terjung - Kool Aid) (MT4648) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/koolaid.bin\n",
      "copying phoenix.bin from ./Roms/ROMS/Phoenix (1983) (Atari - GCC, Michael Feinstein, Patricia Goodson, John Mracek) (CX2673) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/phoenix.bin\n",
      "copying alien.bin from ./Roms/ROMS/Alien (1982) (20th Century Fox Video Games, Douglas 'Dallas North' Neubauer) (11006) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/alien.bin\n",
      "copying demon_attack.bin from ./Roms/ROMS/Demon Attack (Death from Above) (1982) (Imagic, Rob Fulop) (720000-200, 720101-1B, 720101-1C, IA3200, IA3200C, IX-006-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/demon_attack.bin\n",
      "copying crazy_climber.bin from ./Roms/ROMS/Crazy Climber (1983) (Atari - Roklan, Joe Gaucher, Alex Leavens) (CX2683) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/crazy_climber.bin\n",
      "copying video_pinball.bin from ./Roms/ROMS/Pinball (AKA Video Pinball) (Zellers).bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/video_pinball.bin\n",
      "copying yars_revenge.bin from ./Roms/ROMS/Yars' Revenge (Time Freeze) (1982) (Atari, Howard Scott Warshaw - Sears) (CX2655 - 49-75167) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/yars_revenge.bin\n",
      "copying donkey_kong.bin from ./Roms/ROMS/Donkey Kong (1982) (Coleco - Woodside Design Associates - Imaginative Systems Software, Garry Kitchen) (2451) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/donkey_kong.bin\n",
      "copying carnival.bin from ./Roms/ROMS/Carnival (1982) (Coleco - Woodside Design Associates, Steve 'Jessica Stevens' Kitchen) (2468) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/carnival.bin\n",
      "copying defender.bin from ./Roms/ROMS/Defender (1982) (Atari, Robert C. Polaro, Alan J. Murphy - Sears) (CX2609 - 49-75186) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/defender.bin\n",
      "copying gopher.bin from ./Roms/ROMS/Gopher (Gopher Attack) (1982) (U.S. Games Corporation - JWDA, Sylvia Day, Todd Marshall, Robin McDaniel, Henry Will IV) (VC2001) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/gopher.bin\n",
      "copying krull.bin from ./Roms/ROMS/Krull (1983) (Atari, Jerome Domurat, Dave Staugas) (CX2682) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/krull.bin\n",
      "copying pacman.bin from ./Roms/ROMS/Pac-Man (1982) (Atari, Tod Frye) (CX2646) (PAL).bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/pacman.bin\n",
      "copying surround.bin from ./Roms/ROMS/Surround (32 in 1) (Bit Corporation) (R320).bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/surround.bin\n",
      "copying name_this_game.bin from ./Roms/ROMS/Name This Game (Guardians of Treasure, Octopussy) (1983) (U.S. Games Corporation - JWDA, Roger Booth, Sylvia Day, Ron Dubren, Todd Marshall, Robin McDaniel, Wes Trager, Henry Will IV) (VC1007) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/name_this_game.bin\n",
      "copying lost_luggage.bin from ./Roms/ROMS/Lost Luggage (Airport Mayhem) (1982) (Apollo - Games by Apollo, Larry Minor, Ernie Runyon, Ed Salvo) (AP-2004) [no opening scene] ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/lost_luggage.bin\n",
      "copying pong.bin from ./Roms/ROMS/Video Olympics - Pong Sports (Paddle) (1977) (Atari, Joe Decuir - Sears) (CX2621 - 99806, 6-99806, 49-75104) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/pong.bin\n",
      "copying keystone_kapers.bin from ./Roms/ROMS/Keystone Kapers - Raueber und Gendarm (1983) (Activision, Garry Kitchen - Ariola) (EAX-025, EAX-025-04I - 711 025-725) (PAL).bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/keystone_kapers.bin\n",
      "copying kung_fu_master.bin from ./Roms/ROMS/Kung-Fu Master (1987) (Activision - Imagineering, Dan Kitchen, Garry Kitchen) (AG-039-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/kung_fu_master.bin\n",
      "copying trondead.bin from ./Roms/ROMS/TRON - Deadly Discs (TRON Joystick) (1983) (M Network - INTV - APh Technological Consulting, Jeff Ronne, Brett Stutz) (MT5662) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/trondead.bin\n",
      "copying air_raid.bin from ./Roms/ROMS/Air Raid (1982) (Men-A-Vision) (PAL) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/air_raid.bin\n",
      "copying king_kong.bin from ./Roms/ROMS/King Kong (1982) (Tigervision - Software Electronics Corporation, Karl T. Olinger - Teldec) (7-001 - 3.60001 VE) (PAL).bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/king_kong.bin\n",
      "copying qbert.bin from ./Roms/ROMS/Q-bert (1983) (Parker Brothers - Western Technologies, Dave Hampton, Tom Sloper) (PB5360) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/qbert.bin\n",
      "copying robotank.bin from ./Roms/ROMS/Robot Tank (Robotank) (1983) (Activision, Alan Miller) (AZ-028, AG-028-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/robotank.bin\n",
      "copying skiing.bin from ./Roms/ROMS/Skiing - Le Ski (1980) (Activision, Bob Whitehead) (AG-005, CAG-005, AG-005-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/skiing.bin\n",
      "copying hero.bin from ./Roms/ROMS/H.E.R.O. (1984) (Activision, John Van Ryzin) (AZ-036-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/hero.bin\n",
      "copying star_gunner.bin from ./Roms/ROMS/Stargunner (1983) (Telesys, Alex Leavens) (1005) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/star_gunner.bin\n",
      "copying tutankham.bin from ./Roms/ROMS/Tutankham (1983) (Parker Brothers, Dave Engman, Dawn Stockbridge) (PB5340) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/tutankham.bin\n",
      "copying gravitar.bin from ./Roms/ROMS/Gravitar (1983) (Atari, Dan Hitchens, Mimi Nyden) (CX2685) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/gravitar.bin\n",
      "copying montezuma_revenge.bin from ./Roms/ROMS/Montezuma's Revenge - Featuring Panama Joe (1984) (Parker Brothers - JWDA, Henry Will IV) (PB5760) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/montezuma_revenge.bin\n",
      "copying venture.bin from ./Roms/ROMS/Venture (1982) (Coleco, Joseph Biel) (2457) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/venture.bin\n",
      "copying pitfall.bin from ./Roms/ROMS/Pitfall! - Pitfall Harry's Jungle Adventure (Jungle Runner) (1982) (Activision, David Crane) (AX-018, AX-018-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/pitfall.bin\n",
      "copying mr_do.bin from ./Roms/ROMS/Mr. Do! (1983) (CBS Electronics - Individeo, Ed English) (4L4478) (PAL).bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/mr_do.bin\n",
      "copying laser_gates.bin from ./Roms/ROMS/Laser Gates (AKA Innerspace) (1983) (Imagic, Dan Oliver) (720118-2A, 13208, EIX-007-04I) (PAL).bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/laser_gates.bin\n",
      "copying tennis.bin from ./Roms/ROMS/Tennis - Le Tennis (1981) (Activision, Alan Miller) (AG-007, CAG-007) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/tennis.bin\n",
      "copying space_invaders.bin from ./Roms/ROMS/Space Invaders (1980) (Atari, Richard Maurer - Sears) (CX2632 - 49-75153) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/space_invaders.bin\n",
      "copying up_n_down.bin from ./Roms/ROMS/Up 'n Down (1984) (SEGA - Beck-Tech, Steve Beck, Phat Ho) (009-01) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/up_n_down.bin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "copying riverraid.bin from ./Roms/ROMS/River Raid (1982) (Activision, Carol Shaw) (AX-020, AX-020-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/riverraid.bin\n",
      "copying enduro.bin from ./Roms/ROMS/Enduro (1983) (Activision, Larry Miller) (AX-026, AX-026-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/enduro.bin\n",
      "copying amidar.bin from ./Roms/ROMS/Amidar (1982) (Parker Brothers, Ed Temple) (PB5310) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/amidar.bin\n",
      "copying bowling.bin from ./Roms/ROMS/Bowling (1979) (Atari, Larry Kaplan - Sears) (CX2628 - 6-99842, 49-75117) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/bowling.bin\n",
      "copying kangaroo.bin from ./Roms/ROMS/Kangaroo (1983) (Atari - GCC, Patricia Goodson, Josh Littlefield, Kevin Osborn) (CX2689) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/kangaroo.bin\n",
      "copying atlantis.bin from ./Roms/ROMS/Atlantis (Lost City of Atlantis) (1982) (Imagic, Dennis Koble) (720103-1A, 720103-1B, IA3203, IX-010-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/atlantis.bin\n",
      "copying battle_zone.bin from ./Roms/ROMS/Battlezone (1983) (Atari - GCC, Michael Feinstein, Patricia Goodson, Brad Rice) (CX2681) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/battle_zone.bin\n",
      "copying breakout.bin from ./Roms/ROMS/Breakout - Breakaway IV (Paddle) (1978) (Atari, Brad Stewart - Sears) (CX2622 - 6-99813, 49-75107) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/breakout.bin\n",
      "copying jamesbond.bin from ./Roms/ROMS/James Bond 007 (James Bond Agent 007) (1984) (Parker Brothers - On-Time Software, Joe Gaucher, Dan Kurczewski, Louis Marbel, Kathy Von) (PB5110) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/jamesbond.bin\n",
      "copying seaquest.bin from ./Roms/ROMS/Seaquest (1983) (Activision, Steve Cartwright) (AX-022) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/seaquest.bin\n",
      "copying road_runner.bin from patched version of ./Roms/ROMS/Road Runner (1989) (Atari - Bobco, Robert C. Polaro) (CX2663) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/road_runner.bin\n",
      "copying berzerk.bin from ./Roms/ROMS/Berzerk (1982) (Atari, Dan Hitchens - Sears) (CX2650 - 49-75168) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/berzerk.bin\n",
      "copying double_dunk.bin from ./Roms/ROMS/Double Dunk (Super Basketball) (1989) (Atari, Matthew L. Hubbard) (CX26159) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/double_dunk.bin\n",
      "copying centipede.bin from ./Roms/ROMS/Centipede (1983) (Atari - GCC, Patricia Goodson, Josh Littlefield, Douglas B. Macrae) (CX2676) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/centipede.bin\n",
      "copying frostbite.bin from ./Roms/ROMS/Frostbite (Iceman) (1983) (Activision, Steve Cartwright) (AX-031) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/frostbite.bin\n",
      "copying assault.bin from ./Roms/ROMS/Assault (AKA Sky Alien) (1983) (Bomb - Onbase) (CA281).bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/assault.bin\n",
      "copying ice_hockey.bin from ./Roms/ROMS/Ice Hockey - Le Hockey Sur Glace (1981) (Activision, Alan Miller) (AX-012, CAX-012, AX-012-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/ice_hockey.bin\n",
      "copying zaxxon.bin from ./Roms/ROMS/Zaxxon (1983) (Coleco) (2454) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/zaxxon.bin\n",
      "copying elevator_action.bin from ./Roms/ROMS/Elevator Action (1983) (Atari, Dan Hitchens, Dave Staugas) (CX26126) (Prototype) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/elevator_action.bin\n",
      "copying beam_rider.bin from ./Roms/ROMS/Beamrider (1984) (Activision - Cheshire Engineering, David Rolfe, Larry Zwick) (AZ-037-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/beam_rider.bin\n",
      "copying sir_lancelot.bin from ./Roms/ROMS/Sir Lancelot (1983) (Xonox - K-Tel Software - Product Guild, Anthony R. Henderson) (99006, 6220) (PAL).bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/sir_lancelot.bin\n",
      "copying wizard_of_wor.bin from ./Roms/ROMS/Wizard of Wor (1982) (CBS Electronics - Roklan, Joe Hellesen, Joe Wagner) (M8774, M8794) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/wizard_of_wor.bin\n",
      "copying bank_heist.bin from ./Roms/ROMS/Bank Heist (Bonnie & Clyde, Cops 'n' Robbers, Hold-Up, Roaring 20's) (1983) (20th Century Fox Video Games, Bill Aspromonte) (11012) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/bank_heist.bin\n",
      "copying ms_pacman.bin from ./Roms/ROMS/Ms. Pac-Man (1983) (Atari - GCC, Mark S. Ackerman, Patricia Goodson, Josh Littlefield, Douglas B. Macrae, Glenn Parker) (CX2675) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/ms_pacman.bin\n",
      "copying private_eye.bin from ./Roms/ROMS/Private Eye (1984) (Activision, Bob Whitehead) (AG-034-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/private_eye.bin\n",
      "copying journey_escape.bin from ./Roms/ROMS/Journey Escape (1983) (Data Age, J. Ray Dettling) (112-006) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/journey_escape.bin\n",
      "copying asterix.bin from ./Roms/ROMS/Asterix (AKA Taz) (1984) (Atari, Jerome Domurat, Steve Woita) (CX2696).bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/asterix.bin\n",
      "copying boxing.bin from ./Roms/ROMS/Boxing - La Boxe (1980) (Activision, Bob Whitehead) (AG-002, CAG-002, AG-002-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/boxing.bin\n",
      "copying fishing_derby.bin from ./Roms/ROMS/Fishing Derby (1980) (Activision, David Crane) (AG-004) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/fishing_derby.bin\n",
      "copying kaboom.bin from ./Roms/ROMS/Kaboom! (Paddle) (1981) (Activision, Larry Kaplan, David Crane) (AG-010, CAG-010, AG-010-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/kaboom.bin\n",
      "copying freeway.bin from ./Roms/ROMS/Freeway (1981) (Activision, David Crane) (AG-009, AG-009-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/freeway.bin\n",
      "copying pooyan.bin from ./Roms/ROMS/Pooyan (1983) (Konami) (RC 100-X 02) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/pooyan.bin\n",
      "copying asteroids.bin from ./Roms/ROMS/Asteroids (1981) (Atari, Brad Stewart - Sears) (CX2649 - 49-75163) [no copyright] ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/asteroids.bin\n",
      "copying frogger.bin from ./Roms/ROMS/Frogger (1982) (Parker Brothers, Ed English, David Lamkins) (PB5300) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/frogger.bin\n",
      "copying chopper_command.bin from ./Roms/ROMS/Chopper Command (1982) (Activision, Bob Whitehead) (AX-015, AX-015-04) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/chopper_command.bin\n",
      "copying time_pilot.bin from ./Roms/ROMS/Time Pilot (1983) (Coleco - Woodside Design Associates, Harley H. Puthuff Jr.) (2663) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/time_pilot.bin\n",
      "copying galaxian.bin from ./Roms/ROMS/Galaxian (1983) (Atari - GCC, Mark S. Ackerman, Tom Calderwood, Patricia Goodson, Glenn Parker) (CX2684) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/galaxian.bin\n",
      "copying adventure.bin from ./Roms/ROMS/Adventure (1980) (Atari, Warren Robinett) (CX2613, CX2613P) (PAL).bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/adventure.bin\n",
      "copying solaris.bin from ./Roms/ROMS/Solaris (The Last Starfighter, Star Raiders II, Universe) (1986) (Atari, Douglas Neubauer, Mimi Nyden) (CX26136) ~.bin to /Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/atari_py/atari_roms/solaris.bin\n"
     ]
    }
   ],
   "source": [
    "!python -m atari_py.import_roms ./Roms/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/ale_py/roms/utils.py:90: DeprecationWarning: SelectableGroups dict interface is deprecated. Use select.\n",
      "  for external in metadata.entry_points().get(self.group, []):\n",
      "/Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/gym/envs/registration.py:424: UserWarning: \u001b[33mWARN: Custom namespace `ALE` is being overridden by namespace `ALE`. If you are developing a plugin you shouldn't specify a namespace in `register` calls. The namespace is specified through the entry point package metadata.\u001b[0m\n",
      "  f\"Custom namespace `{spec.namespace}` is being overridden \"\n",
      "Warning: Gym version v0.24.0 has a number of critical issues with `gym.make` such that the `reset` and `step` functions are called before returning the environment. It is recommend to downgrading to v0.23.1 or upgrading to v0.25.1\n"
     ]
    }
   ],
   "source": [
    "# general imports\n",
    "import cv2\n",
    "import math\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "# reinforcement learning related imports\n",
    "import re\n",
    "import atari_py as ap\n",
    "from collections import deque\n",
    "from gym import make, ObservationWrapper, Wrapper\n",
    "from gym.spaces import Box\n",
    "\n",
    "# pytorch imports \n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import save\n",
    "from torch.optim import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvDQN(nn.Module):\n",
    "    def __init__(self, ip_sz, tot_num_acts):\n",
    "        super(ConvDQN, self).__init__()\n",
    "        self._ip_sz = ip_sz\n",
    "        self._tot_num_acts = tot_num_acts\n",
    "\n",
    "        self.cnv1 = nn.Conv2d(ip_sz[0], 32, kernel_size=8, stride=4)\n",
    "        self.activation = nn.ReLU()\n",
    "        self.cnv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)\n",
    "        self.cnv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)\n",
    "        self.fc1 = nn.Linear(self.feat_sz, 512)\n",
    "        self.fc2 = nn.Linear(512, tot_num_acts)\n",
    "\n",
    "    def forward(self, x):\n",
    "        op = self.cnv1(x)\n",
    "        op = self.activation(op)\n",
    "        op = self.cnv2(op)\n",
    "        op = self.activation(op)\n",
    "        op = self.cnv3(op)\n",
    "        op = self.activation(op).view(x.size()[0], -1)\n",
    "        op = self.fc1(op)\n",
    "        op = self.activation(op)\n",
    "        op = self.fc2(op)\n",
    "        return op\n",
    "\n",
    "    @property\n",
    "    def feat_sz(self):\n",
    "        x = torch.zeros(1, *self._ip_sz)\n",
    "        x = self.cnv1(x)\n",
    "        x = self.activation(x)\n",
    "        x = self.cnv2(x)\n",
    "        x = self.activation(x)\n",
    "        x = self.cnv3(x)\n",
    "        x = self.activation(x)\n",
    "        return x.view(1, -1).size(1)\n",
    "\n",
    "    def perf_action(self, stt, eps, dvc):\n",
    "        if random.random() > eps:\n",
    "            stt = torch.from_numpy(np.float32(stt)).unsqueeze(0).to(dvc)\n",
    "            q_val = self.forward(stt)\n",
    "            act = q_val.max(1)[1].item()\n",
    "        else:\n",
    "            act = random.randrange(self._tot_num_acts)\n",
    "        return act"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_temp_diff_loss(mdl, tgt_mdl, bch, gm, dvc):\n",
    "    st, act, rwd, nxt_st, fin = bch\n",
    "\n",
    "    st = torch.from_numpy(np.float32(st)).to(dvc)\n",
    "    nxt_st = torch.from_numpy(np.float32(nxt_st)).to(dvc)\n",
    "    act = torch.from_numpy(act).to(dvc)\n",
    "    rwd = torch.from_numpy(rwd).to(dvc)\n",
    "    fin = torch.from_numpy(fin).to(dvc)\n",
    "\n",
    "    q_vals = mdl(st)\n",
    "    nxt_q_vals = tgt_mdl(nxt_st)\n",
    "\n",
    "    q_val = q_vals.gather(1, act.unsqueeze(-1)).squeeze(-1)\n",
    "    nxt_q_val = nxt_q_vals.max(1)[0]\n",
    "    exp_q_val = rwd + gm * nxt_q_val * (1 - fin)\n",
    "\n",
    "    loss = (q_val - exp_q_val.data.to(dvc)).pow(2).mean()\n",
    "    loss.backward()\n",
    "\n",
    "\n",
    "def upd_eps(epd):\n",
    "    last_eps = EPS_FINL\n",
    "    first_eps = EPS_STRT\n",
    "    eps_decay = EPS_DECAY\n",
    "    eps = last_eps + (first_eps - last_eps) * math.exp(-1 * ((epd + 1) / eps_decay))\n",
    "    return eps\n",
    "\n",
    "\n",
    "def models_init(env, dvc):\n",
    "    mdl = ConvDQN(env.observation_space.shape, env.action_space.n).to(dvc)\n",
    "    tgt_mdl = ConvDQN(env.observation_space.shape, env.action_space.n).to(dvc)\n",
    "    return mdl, tgt_mdl\n",
    "\n",
    "\n",
    "def gym_to_atari_format(gym_env):\n",
    "    return re.sub(r\"(?<!^)(?=[A-Z])\", \"_\", gym_env).lower()\n",
    "\n",
    "\n",
    "def check_atari_env(env):\n",
    "    for f in [\"Deterministic\", \"ramDeterministic\", \"ram\", \"NoFrameskip\", \"ramNoFrameSkip\"]:\n",
    "        env = env.replace(f, \"\")\n",
    "    env = re.sub(r\"-v\\d+\", \"\", env)\n",
    "    env = gym_to_atari_format(env) \n",
    "    return True if env in ap.list_games() else False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RepBfr:\n",
    "    def __init__(self, cap_max):\n",
    "        self._bfr = deque(maxlen=cap_max)\n",
    "\n",
    "    def push(self, st, act, rwd, nxt_st, fin):\n",
    "        self._bfr.append((st, act, rwd, nxt_st, fin))\n",
    "\n",
    "    def smpl(self, bch_sz):\n",
    "        idxs = np.random.choice(len(self._bfr), bch_sz, False)\n",
    "        bch = zip(*[self._bfr[i] for i in idxs])\n",
    "        st, act, rwd, nxt_st, fin = bch\n",
    "        return (np.array(st), np.array(act), np.array(rwd, dtype=np.float32),\n",
    "                np.array(nxt_st), np.array(fin, dtype=np.uint8))\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self._bfr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TrMetadata:\n",
    "    def __init__(self):\n",
    "        self._avg = 0.0\n",
    "        self._bst_rwd = -float(\"inf\")\n",
    "        self._bst_avg = -float(\"inf\")\n",
    "        self._rwds = []\n",
    "        self._avg_rng = 100\n",
    "        self._idx = 0\n",
    "\n",
    "    @property\n",
    "    def bst_rwd(self):\n",
    "        return self._bst_rwd\n",
    "\n",
    "    @property\n",
    "    def bst_avg(self):\n",
    "        return self._bst_avg\n",
    "\n",
    "    @property\n",
    "    def avg(self):\n",
    "        avg_rng = self._avg_rng * -1\n",
    "        return sum(self._rwds[avg_rng:]) / len(self._rwds[avg_rng:])\n",
    "\n",
    "    @property\n",
    "    def idx(self):\n",
    "        return self._idx\n",
    "\n",
    "    def _upd_bst_rwd(self, epd_rwd):\n",
    "        if epd_rwd > self.bst_rwd:\n",
    "            self._bst_rwd = epd_rwd\n",
    "\n",
    "    def _upd_bst_avg(self):\n",
    "        if self.avg > self.bst_avg:\n",
    "            self._bst_avg = self.avg\n",
    "            return True\n",
    "        return False\n",
    "\n",
    "    def upd_rwds(self, epd_rwd):\n",
    "        self._rwds.append(epd_rwd)\n",
    "        self._upd_bst_rwd(epd_rwd)\n",
    "        return self._upd_bst_avg()\n",
    "\n",
    "    def upd_idx(self):\n",
    "        self._idx += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ClassicControl(Wrapper):\n",
    "    def __init__(self, env, is_atari):\n",
    "        super(ClassicControl, self).__init__(env)\n",
    "        self._is_atari = is_atari\n",
    "\n",
    "    def reset(self):\n",
    "        if self._is_atari:\n",
    "            return self.env.reset()\n",
    "        else:\n",
    "            self.env.reset()\n",
    "            return self.env.render(mode=\"rgb_array\")\n",
    "\n",
    "\n",
    "class FrameDownSample(ObservationWrapper):\n",
    "    def __init__(self, env):\n",
    "        super(FrameDownSample, self).__init__(env)\n",
    "        self.observation_space = Box(low=0, high=255, shape=(84, 84, 1), dtype=np.uint8)\n",
    "        self._width = 84\n",
    "        self._height = 84\n",
    "\n",
    "    def observation(self, observation):\n",
    "        frame = cv2.cvtColor(observation, cv2.COLOR_RGB2GRAY)\n",
    "        frame = cv2.resize(frame, (self._width, self._height), interpolation=cv2.INTER_AREA)\n",
    "        return frame[:, :, None]\n",
    "\n",
    "\n",
    "class MaxAndSkipEnv(Wrapper):\n",
    "    def __init__(self, env, atari, skip=4):\n",
    "        super(MaxAndSkipEnv, self).__init__(env)\n",
    "        self._obs_buffer = deque(maxlen=2)\n",
    "        self._skip = skip\n",
    "        self._atari = atari\n",
    "\n",
    "    def step(self, act):\n",
    "        total_rwd = 0.0\n",
    "        fin = None\n",
    "        for _ in range(self._skip):\n",
    "            obs, rwd, fin, log = self.env.step(act)\n",
    "            if not self._atari:\n",
    "                obs = self.env.render(mode=\"rgb_array\")\n",
    "            self._obs_buffer.append(obs)\n",
    "            total_rwd += rwd\n",
    "            if fin:\n",
    "                break\n",
    "        max_frame = np.max(np.stack(self._obs_buffer), axis=0)\n",
    "        return max_frame, total_rwd, fin, log\n",
    "\n",
    "    def reset(self):\n",
    "        self._obs_buffer.clear()\n",
    "        obs = self.env.reset()\n",
    "        self._obs_buffer.append(obs)\n",
    "        return obs\n",
    "\n",
    "\n",
    "class FrameResetEnv(Wrapper):\n",
    "    def __init__(self, env):\n",
    "        Wrapper.__init__(self, env)\n",
    "        if len(env.unwrapped.get_action_meanings()) < 3:\n",
    "            raise ValueError(\"min required action space of 3!\")\n",
    "\n",
    "    def reset(self, **kwargs):\n",
    "        self.env.reset(**kwargs)\n",
    "        obs, _, fin, _ = self.env.step(1)\n",
    "        if fin:\n",
    "            self.env.reset(**kwargs)\n",
    "        obs, _, fin, _ = self.env.step(2)\n",
    "        if fin:\n",
    "            self.env.reset(**kwargs)\n",
    "        return obs\n",
    "\n",
    "    def step(self, act):\n",
    "        return self.env.step(act)\n",
    "\n",
    "\n",
    "class FrameBuffer(ObservationWrapper):\n",
    "    def __init__(self, env, num_steps, dtype=np.float32):\n",
    "        super(FrameBuffer, self).__init__(env)\n",
    "        obs_space = env.observation_space\n",
    "        self._dtype = dtype\n",
    "        self.observation_space = Box(obs_space.low.repeat(num_steps, axis=0),\n",
    "                                     obs_space.high.repeat(num_steps, axis=0), dtype=self._dtype)\n",
    "\n",
    "    def reset(self):\n",
    "        self.buffer = np.zeros_like(self.observation_space.low, dtype=self._dtype)\n",
    "        return self.observation(self.env.reset())\n",
    "\n",
    "    def observation(self, observation):\n",
    "        self.buffer[:-1] = self.buffer[1:]\n",
    "        self.buffer[-1] = observation\n",
    "        return self.buffer\n",
    "\n",
    "\n",
    "class Image2PyTorch(ObservationWrapper):\n",
    "    def __init__(self, env):\n",
    "        super(Image2PyTorch, self).__init__(env)\n",
    "        obs_shape = self.observation_space.shape\n",
    "        self.observation_space = Box(low=0.0, high=1.0, shape=(obs_shape[::-1]), dtype=np.float32)\n",
    "\n",
    "    def observation(self, observation):\n",
    "        return np.moveaxis(observation, 2, 0)\n",
    "\n",
    "\n",
    "class NormalizeFloats(ObservationWrapper):\n",
    "    def observation(self, obs):\n",
    "        return np.array(obs).astype(np.float32) / 255.0\n",
    "        \n",
    "\n",
    "def wrap_env(env_ip):\n",
    "    env = make(env_ip)\n",
    "    is_atari = check_atari_env(env_ip)\n",
    "    env = ClassicControl(env, is_atari)\n",
    "    env = MaxAndSkipEnv(env, is_atari)\n",
    "    try:\n",
    "        env_acts = env.unwrapped.get_action_meanings()\n",
    "        if \"FIRE\" in env_acts:\n",
    "            env = FrameResetEnv(env)\n",
    "    except AttributeError:\n",
    "        pass\n",
    "    env = FrameDownSample(env)\n",
    "    env = Image2PyTorch(env)\n",
    "    env = FrameBuffer(env, 4)\n",
    "    env = NormalizeFloats(env)\n",
    "    return env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def upd_grph(mdl, tgt_mdl, opt, rpl_bfr, dvc, log):\n",
    "    if len(rpl_bfr) > INIT_LEARN:\n",
    "        if not log.idx % TGT_UPD_FRQ:\n",
    "            tgt_mdl.load_state_dict(mdl.state_dict())\n",
    "        opt.zero_grad()\n",
    "        bch = rpl_bfr.smpl(B_S)\n",
    "        calc_temp_diff_loss(mdl, tgt_mdl, bch, G, dvc)\n",
    "        opt.step()\n",
    "\n",
    "\n",
    "def fin_epsd(mdl, env, log, epd_rwd, epd, eps):\n",
    "    bst_so_fat = log.upd_rwds(epd_rwd)\n",
    "    if bst_so_fat:\n",
    "        print(f\"checkpointing current model weights. highest running_average_reward of\\\n",
    " {round(log.bst_avg, 3)} achieved!\")\n",
    "        save(mdl.state_dict(), f\"{env}.dat\")\n",
    "    print(f\"episode_num {epd}, curr_reward: {epd_rwd}, best_reward: {log.bst_rwd},\\\n",
    " running_avg_reward: {round(log.avg, 3)}, curr_epsilon: {round(eps, 4)}\")\n",
    "\n",
    "\n",
    "def run_epsd(env, mdl, tgt_mdl, opt, rpl_bfr, dvc, log, epd):\n",
    "    epd_rwd = 0.0\n",
    "    st = env.reset()\n",
    "\n",
    "    while True:\n",
    "        eps = upd_eps(log.idx)\n",
    "        act = mdl.perf_action(st, eps, dvc)\n",
    "        if True:\n",
    "            env.render(mode=\"rgb_array\")\n",
    "        nxt_st, rwd, fin, _ = env.step(act)\n",
    "        rpl_bfr.push(st, act, rwd, nxt_st, fin)\n",
    "        st = nxt_st\n",
    "        epd_rwd += rwd\n",
    "        log.upd_idx()\n",
    "        upd_grph(mdl, tgt_mdl, opt, rpl_bfr, dvc, log)\n",
    "        if fin:\n",
    "            fin_epsd(mdl, ENV, log, epd_rwd, epd, eps)\n",
    "            break\n",
    "\n",
    "\n",
    "def train(env, mdl, tgt_mdl, opt, rpl_bfr, dvc):\n",
    "    log = TrMetadata()\n",
    "\n",
    "    for epd in range(N_EPDS):\n",
    "        run_epsd(env, mdl, tgt_mdl, opt, rpl_bfr, dvc, log, epd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "B_S = 64\n",
    "ENV = \"Pong-v4\"\n",
    "EPS_STRT = 1.0\n",
    "EPS_FINL = 0.005\n",
    "EPS_DECAY = 100000\n",
    "G = 0.99\n",
    "INIT_LEARN = 10000\n",
    "LR = 1e-4\n",
    "MEM_CAP = 20000\n",
    "N_EPDS = 50000\n",
    "TGT_UPD_FRQ = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A.L.E: Arcade Learning Environment (version +978d2ce)\n",
      "[Powered by Stella]\n",
      "/Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/gym/utils/seeding.py:160: DeprecationWarning: \u001b[33mWARN: Function `hash_seed(seed, max_bytes)` is marked as deprecated and will be removed in the future. \u001b[0m\n",
      "  \"Function `hash_seed(seed, max_bytes)` is marked as deprecated and will be removed in the future. \"\n",
      "/Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/gym/utils/seeding.py:204: DeprecationWarning: \u001b[33mWARN: Function `_bigint_from_bytes(bytes)` is marked as deprecated and will be removed in the future. \u001b[0m\n",
      "  \"Function `_bigint_from_bytes(bytes)` is marked as deprecated and will be removed in the future. \"\n",
      "/Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/gym/utils/seeding.py:64: DeprecationWarning: \u001b[33mWARN: Function `rng.randint(low, [high, size, dtype])` is marked as deprecated and will be removed in the future. Please use `rng.integers(low, [high, size, dtype])` instead.\u001b[0m\n",
      "  \"Function `rng.randint(low, [high, size, dtype])` is marked as deprecated \"\n",
      "/Users/Ashish.Jha/anaconda3/envs/python37/lib/python3.7/site-packages/gym/envs/registration.py:620: UserWarning: \u001b[33mWARN: Env check failed with the following message: The environment cannot be reset with a random seed, even though `seed` or `kwargs` appear in the signature. This should never happen, please report this issue. The error was: reset() got an unexpected keyword argument 'seed'\n",
      "You can set `disable_env_checker=True` to disable this check.\u001b[0m\n",
      "  f\"Env check failed with the following message: {e}\\n\"\n",
      "[W NNPACK.cpp:51] Could not initialize NNPACK! Reason: Unsupported hardware.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpointing current model weights. highest running_average_reward of -21.0 achieved!\n",
      "episode_num 0, curr_reward: -21.0, best_reward: -21.0, running_avg_reward: -21.0, curr_epsilon: 0.9974\n",
      "episode_num 1, curr_reward: -21.0, best_reward: -21.0, running_avg_reward: -21.0, curr_epsilon: 0.9948\n",
      "episode_num 2, curr_reward: -21.0, best_reward: -21.0, running_avg_reward: -21.0, curr_epsilon: 0.9919\n",
      "episode_num 3, curr_reward: -21.0, best_reward: -21.0, running_avg_reward: -21.0, curr_epsilon: 0.9888\n",
      "checkpointing current model weights. highest running_average_reward of -20.8 achieved!\n",
      "episode_num 4, curr_reward: -20.0, best_reward: -20.0, running_avg_reward: -20.8, curr_epsilon: 0.9855\n",
      "episode_num 5, curr_reward: -21.0, best_reward: -20.0, running_avg_reward: -20.833, curr_epsilon: 0.983\n",
      "checkpointing current model weights. highest running_average_reward of -20.714 achieved!\n",
      "episode_num 6, curr_reward: -20.0, best_reward: -20.0, running_avg_reward: -20.714, curr_epsilon: 0.98\n",
      "episode_num 7, curr_reward: -21.0, best_reward: -20.0, running_avg_reward: -20.75, curr_epsilon: 0.977\n",
      "episode_num 8, curr_reward: -21.0, best_reward: -20.0, running_avg_reward: -20.778, curr_epsilon: 0.9744\n",
      "checkpointing current model weights. highest running_average_reward of -20.6 achieved!\n",
      "episode_num 9, curr_reward: -19.0, best_reward: -19.0, running_avg_reward: -20.6, curr_epsilon: 0.9712\n",
      "episode_num 10, curr_reward: -21.0, best_reward: -19.0, running_avg_reward: -20.636, curr_epsilon: 0.9678\n",
      "episode_num 11, curr_reward: -21.0, best_reward: -19.0, running_avg_reward: -20.667, curr_epsilon: 0.9654\n",
      "episode_num 12, curr_reward: -21.0, best_reward: -19.0, running_avg_reward: -20.692, curr_epsilon: 0.9628\n",
      "episode_num 13, curr_reward: -21.0, best_reward: -19.0, running_avg_reward: -20.714, curr_epsilon: 0.9603\n",
      "checkpointing current model weights. highest running_average_reward of -20.467 achieved!\n",
      "episode_num 14, curr_reward: -17.0, best_reward: -17.0, running_avg_reward: -20.467, curr_epsilon: 0.9566\n",
      "checkpointing current model weights. highest running_average_reward of -20.438 achieved!\n",
      "episode_num 15, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.438, curr_epsilon: 0.9538\n",
      "checkpointing current model weights. highest running_average_reward of -20.412 achieved!\n",
      "episode_num 16, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.412, curr_epsilon: 0.9505\n",
      "checkpointing current model weights. highest running_average_reward of -20.278 achieved!\n",
      "episode_num 17, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -20.278, curr_epsilon: 0.947\n",
      "episode_num 18, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.316, curr_epsilon: 0.9444\n",
      "episode_num 19, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.3, curr_epsilon: 0.9414\n",
      "episode_num 20, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.286, curr_epsilon: 0.9388\n",
      "checkpointing current model weights. highest running_average_reward of -20.273 achieved!\n",
      "episode_num 21, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.273, curr_epsilon: 0.9362\n",
      "episode_num 22, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.304, curr_epsilon: 0.9335\n",
      "episode_num 23, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.333, curr_epsilon: 0.931\n",
      "episode_num 24, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.32, curr_epsilon: 0.9284\n",
      "episode_num 25, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.308, curr_epsilon: 0.9253\n",
      "episode_num 26, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.333, curr_epsilon: 0.9229\n",
      "episode_num 27, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.357, curr_epsilon: 0.9204\n",
      "episode_num 28, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.379, curr_epsilon: 0.9181\n",
      "episode_num 29, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.4, curr_epsilon: 0.9153\n",
      "episode_num 30, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.419, curr_epsilon: 0.9127\n",
      "episode_num 31, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.438, curr_epsilon: 0.9103\n",
      "episode_num 32, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.455, curr_epsilon: 0.9076\n",
      "episode_num 33, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.441, curr_epsilon: 0.9049\n",
      "episode_num 34, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.457, curr_epsilon: 0.9023\n",
      "episode_num 35, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.472, curr_epsilon: 0.8998\n",
      "episode_num 36, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.486, curr_epsilon: 0.8975\n",
      "episode_num 37, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.5, curr_epsilon: 0.895\n",
      "episode_num 38, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.487, curr_epsilon: 0.8925\n",
      "episode_num 39, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.5, curr_epsilon: 0.8898\n",
      "episode_num 40, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.512, curr_epsilon: 0.8875\n",
      "episode_num 41, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.524, curr_epsilon: 0.8851\n",
      "episode_num 42, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.535, curr_epsilon: 0.8827\n",
      "episode_num 43, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.545, curr_epsilon: 0.8803\n",
      "episode_num 44, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.556, curr_epsilon: 0.8779\n",
      "episode_num 45, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.522, curr_epsilon: 0.8752\n",
      "episode_num 46, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.511, curr_epsilon: 0.8724\n",
      "episode_num 47, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.521, curr_epsilon: 0.8697\n",
      "episode_num 48, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.51, curr_epsilon: 0.8674\n",
      "episode_num 49, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.52, curr_epsilon: 0.8649\n",
      "episode_num 50, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.51, curr_epsilon: 0.8625\n",
      "episode_num 51, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.519, curr_epsilon: 0.8599\n",
      "episode_num 52, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.528, curr_epsilon: 0.8573\n",
      "episode_num 53, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.519, curr_epsilon: 0.8545\n",
      "episode_num 54, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.527, curr_epsilon: 0.8519\n",
      "episode_num 55, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.518, curr_epsilon: 0.8491\n",
      "episode_num 56, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.526, curr_epsilon: 0.8467\n",
      "episode_num 57, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.517, curr_epsilon: 0.8439\n",
      "episode_num 58, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.508, curr_epsilon: 0.8413\n",
      "episode_num 59, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.517, curr_epsilon: 0.8387\n",
      "episode_num 60, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.492, curr_epsilon: 0.8358\n",
      "episode_num 61, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.5, curr_epsilon: 0.8333\n",
      "episode_num 62, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.492, curr_epsilon: 0.8306\n",
      "episode_num 63, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.5, curr_epsilon: 0.8282\n",
      "episode_num 64, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.508, curr_epsilon: 0.8254\n",
      "episode_num 65, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.5, curr_epsilon: 0.8229\n",
      "episode_num 66, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.493, curr_epsilon: 0.8202\n",
      "episode_num 67, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.5, curr_epsilon: 0.8179\n",
      "episode_num 68, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.507, curr_epsilon: 0.815\n",
      "episode_num 69, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.514, curr_epsilon: 0.8122\n",
      "episode_num 70, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.521, curr_epsilon: 0.8096\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 71, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.528, curr_epsilon: 0.8067\n",
      "episode_num 72, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.534, curr_epsilon: 0.8043\n",
      "episode_num 73, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.514, curr_epsilon: 0.8006\n",
      "episode_num 74, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.507, curr_epsilon: 0.7983\n",
      "episode_num 75, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.513, curr_epsilon: 0.7959\n",
      "episode_num 76, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.519, curr_epsilon: 0.7931\n",
      "episode_num 77, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.5, curr_epsilon: 0.7905\n",
      "episode_num 78, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.506, curr_epsilon: 0.7881\n",
      "episode_num 79, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.5, curr_epsilon: 0.7855\n",
      "episode_num 80, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.494, curr_epsilon: 0.7833\n",
      "episode_num 81, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.5, curr_epsilon: 0.7805\n",
      "episode_num 82, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.494, curr_epsilon: 0.7781\n",
      "episode_num 83, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.5, curr_epsilon: 0.7753\n",
      "episode_num 84, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.506, curr_epsilon: 0.7731\n",
      "episode_num 85, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.5, curr_epsilon: 0.7705\n",
      "episode_num 86, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.506, curr_epsilon: 0.7684\n",
      "episode_num 87, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.511, curr_epsilon: 0.7661\n",
      "episode_num 88, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.517, curr_epsilon: 0.7639\n",
      "episode_num 89, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.522, curr_epsilon: 0.762\n",
      "episode_num 90, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.527, curr_epsilon: 0.7597\n",
      "episode_num 91, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.533, curr_epsilon: 0.7571\n",
      "episode_num 92, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.527, curr_epsilon: 0.7542\n",
      "episode_num 93, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.521, curr_epsilon: 0.7516\n",
      "episode_num 94, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.526, curr_epsilon: 0.7494\n",
      "episode_num 95, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.531, curr_epsilon: 0.7471\n",
      "episode_num 96, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.536, curr_epsilon: 0.745\n",
      "episode_num 97, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.52, curr_epsilon: 0.7423\n",
      "episode_num 98, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.515, curr_epsilon: 0.7396\n",
      "episode_num 99, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.5, curr_epsilon: 0.7368\n",
      "episode_num 100, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.5, curr_epsilon: 0.7343\n",
      "episode_num 101, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -20.47, curr_epsilon: 0.7314\n",
      "episode_num 102, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.47, curr_epsilon: 0.7287\n",
      "episode_num 103, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.47, curr_epsilon: 0.7266\n",
      "episode_num 104, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.47, curr_epsilon: 0.7243\n",
      "episode_num 105, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.45, curr_epsilon: 0.7215\n",
      "episode_num 106, curr_reward: -17.0, best_reward: -17.0, running_avg_reward: -20.42, curr_epsilon: 0.7184\n",
      "episode_num 107, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.42, curr_epsilon: 0.7161\n",
      "episode_num 108, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.4, curr_epsilon: 0.7137\n",
      "episode_num 109, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.41, curr_epsilon: 0.7112\n",
      "episode_num 110, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.41, curr_epsilon: 0.7092\n",
      "episode_num 111, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.4, curr_epsilon: 0.7071\n",
      "episode_num 112, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.4, curr_epsilon: 0.7046\n",
      "episode_num 113, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.39, curr_epsilon: 0.7023\n",
      "episode_num 114, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.42, curr_epsilon: 0.7002\n",
      "episode_num 115, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.41, curr_epsilon: 0.6979\n",
      "episode_num 116, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.42, curr_epsilon: 0.6953\n",
      "episode_num 117, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.45, curr_epsilon: 0.6929\n",
      "episode_num 118, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.43, curr_epsilon: 0.6904\n",
      "episode_num 119, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.44, curr_epsilon: 0.6885\n",
      "episode_num 120, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.44, curr_epsilon: 0.6864\n",
      "episode_num 121, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.45, curr_epsilon: 0.6845\n",
      "episode_num 122, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.45, curr_epsilon: 0.6824\n",
      "episode_num 123, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.44, curr_epsilon: 0.6798\n",
      "episode_num 124, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.44, curr_epsilon: 0.6775\n",
      "episode_num 125, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.45, curr_epsilon: 0.6753\n",
      "episode_num 126, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.45, curr_epsilon: 0.673\n",
      "episode_num 127, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.45, curr_epsilon: 0.6712\n",
      "episode_num 128, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.44, curr_epsilon: 0.669\n",
      "episode_num 129, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.42, curr_epsilon: 0.6665\n",
      "episode_num 130, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.41, curr_epsilon: 0.6643\n",
      "episode_num 131, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.39, curr_epsilon: 0.6616\n",
      "episode_num 132, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.39, curr_epsilon: 0.6593\n",
      "episode_num 133, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.4, curr_epsilon: 0.6574\n",
      "episode_num 134, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.4, curr_epsilon: 0.6554\n",
      "episode_num 135, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.4, curr_epsilon: 0.6536\n",
      "episode_num 136, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.4, curr_epsilon: 0.6517\n",
      "episode_num 137, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.39, curr_epsilon: 0.6493\n",
      "episode_num 138, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -20.37, curr_epsilon: 0.6461\n",
      "episode_num 139, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.35, curr_epsilon: 0.6437\n",
      "episode_num 140, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -20.32, curr_epsilon: 0.6413\n",
      "episode_num 141, curr_reward: -17.0, best_reward: -17.0, running_avg_reward: -20.28, curr_epsilon: 0.6386\n",
      "checkpointing current model weights. highest running_average_reward of -20.25 achieved!\n",
      "episode_num 142, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -20.25, curr_epsilon: 0.6361\n",
      "episode_num 143, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.25, curr_epsilon: 0.6338\n",
      "episode_num 144, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.25, curr_epsilon: 0.6321\n",
      "episode_num 145, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.26, curr_epsilon: 0.63\n",
      "episode_num 146, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.26, curr_epsilon: 0.6276\n",
      "checkpointing current model weights. highest running_average_reward of -20.23 achieved!\n",
      "episode_num 147, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -20.23, curr_epsilon: 0.625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 148, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.24, curr_epsilon: 0.6227\n",
      "checkpointing current model weights. highest running_average_reward of -20.22 achieved!\n",
      "episode_num 149, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.22, curr_epsilon: 0.6205\n",
      "episode_num 150, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.23, curr_epsilon: 0.6187\n",
      "episode_num 151, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.23, curr_epsilon: 0.6166\n",
      "episode_num 152, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.22, curr_epsilon: 0.6142\n",
      "episode_num 153, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.23, curr_epsilon: 0.6122\n",
      "episode_num 154, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.23, curr_epsilon: 0.6101\n",
      "episode_num 155, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.23, curr_epsilon: 0.6079\n",
      "checkpointing current model weights. highest running_average_reward of -20.19 achieved!\n",
      "episode_num 156, curr_reward: -17.0, best_reward: -17.0, running_avg_reward: -20.19, curr_epsilon: 0.6048\n",
      "checkpointing current model weights. highest running_average_reward of -20.17 achieved!\n",
      "episode_num 157, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -20.17, curr_epsilon: 0.6022\n",
      "episode_num 158, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.18, curr_epsilon: 0.5999\n",
      "checkpointing current model weights. highest running_average_reward of -20.15 achieved!\n",
      "episode_num 159, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -20.15, curr_epsilon: 0.5973\n",
      "episode_num 160, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.17, curr_epsilon: 0.5956\n",
      "episode_num 161, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.15, curr_epsilon: 0.5929\n",
      "episode_num 162, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.16, curr_epsilon: 0.5908\n",
      "episode_num 163, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.15, curr_epsilon: 0.5883\n",
      "episode_num 164, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.15, curr_epsilon: 0.5859\n",
      "episode_num 165, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.16, curr_epsilon: 0.5839\n",
      "episode_num 166, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.16, curr_epsilon: 0.5815\n",
      "episode_num 167, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.15, curr_epsilon: 0.5791\n",
      "episode_num 168, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.15, curr_epsilon: 0.5767\n",
      "checkpointing current model weights. highest running_average_reward of -20.14 achieved!\n",
      "episode_num 169, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.14, curr_epsilon: 0.5747\n",
      "episode_num 170, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.14, curr_epsilon: 0.5729\n",
      "checkpointing current model weights. highest running_average_reward of -20.12 achieved!\n",
      "episode_num 171, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.12, curr_epsilon: 0.5704\n",
      "checkpointing current model weights. highest running_average_reward of -20.11 achieved!\n",
      "episode_num 172, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.11, curr_epsilon: 0.568\n",
      "episode_num 173, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.11, curr_epsilon: 0.5658\n",
      "episode_num 174, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.11, curr_epsilon: 0.5634\n",
      "checkpointing current model weights. highest running_average_reward of -20.08 achieved!\n",
      "episode_num 175, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -20.08, curr_epsilon: 0.5609\n",
      "checkpointing current model weights. highest running_average_reward of -20.05 achieved!\n",
      "episode_num 176, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -20.05, curr_epsilon: 0.5587\n",
      "checkpointing current model weights. highest running_average_reward of -20.04 achieved!\n",
      "episode_num 177, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -20.04, curr_epsilon: 0.5562\n",
      "checkpointing current model weights. highest running_average_reward of -20.02 achieved!\n",
      "episode_num 178, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -20.02, curr_epsilon: 0.5537\n",
      "episode_num 179, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -20.03, curr_epsilon: 0.5516\n",
      "checkpointing current model weights. highest running_average_reward of -20.01 achieved!\n",
      "episode_num 180, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -20.01, curr_epsilon: 0.5497\n",
      "checkpointing current model weights. highest running_average_reward of -20.0 achieved!\n",
      "episode_num 181, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -20.0, curr_epsilon: 0.5476\n",
      "checkpointing current model weights. highest running_average_reward of -19.99 achieved!\n",
      "episode_num 182, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -19.99, curr_epsilon: 0.5453\n",
      "episode_num 183, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -19.99, curr_epsilon: 0.5433\n",
      "checkpointing current model weights. highest running_average_reward of -19.97 achieved!\n",
      "episode_num 184, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -19.97, curr_epsilon: 0.5415\n",
      "episode_num 185, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -19.97, curr_epsilon: 0.5393\n",
      "checkpointing current model weights. highest running_average_reward of -19.93 achieved!\n",
      "episode_num 186, curr_reward: -17.0, best_reward: -17.0, running_avg_reward: -19.93, curr_epsilon: 0.5368\n",
      "checkpointing current model weights. highest running_average_reward of -19.89 achieved!\n",
      "episode_num 187, curr_reward: -17.0, best_reward: -17.0, running_avg_reward: -19.89, curr_epsilon: 0.5344\n",
      "checkpointing current model weights. highest running_average_reward of -19.86 achieved!\n",
      "episode_num 188, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -19.86, curr_epsilon: 0.5323\n",
      "checkpointing current model weights. highest running_average_reward of -19.84 achieved!\n",
      "episode_num 189, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -19.84, curr_epsilon: 0.5298\n",
      "checkpointing current model weights. highest running_average_reward of -19.81 achieved!\n",
      "episode_num 190, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -19.81, curr_epsilon: 0.5269\n",
      "checkpointing current model weights. highest running_average_reward of -19.8 achieved!\n",
      "episode_num 191, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -19.8, curr_epsilon: 0.5252\n",
      "checkpointing current model weights. highest running_average_reward of -19.78 achieved!\n",
      "episode_num 192, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -19.78, curr_epsilon: 0.5229\n",
      "checkpointing current model weights. highest running_average_reward of -19.77 achieved!\n",
      "episode_num 193, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -19.77, curr_epsilon: 0.5208\n",
      "checkpointing current model weights. highest running_average_reward of -19.74 achieved!\n",
      "episode_num 194, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -19.74, curr_epsilon: 0.5186\n",
      "episode_num 195, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -19.74, curr_epsilon: 0.5165\n",
      "episode_num 196, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -19.74, curr_epsilon: 0.5148\n",
      "episode_num 197, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -19.74, curr_epsilon: 0.5127\n",
      "checkpointing current model weights. highest running_average_reward of -19.72 achieved!\n",
      "episode_num 198, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -19.72, curr_epsilon: 0.5103\n",
      "episode_num 199, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -19.74, curr_epsilon: 0.5081\n",
      "episode_num 200, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -19.72, curr_epsilon: 0.5058\n",
      "episode_num 201, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -19.75, curr_epsilon: 0.5039\n",
      "episode_num 202, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -19.74, curr_epsilon: 0.5022\n",
      "checkpointing current model weights. highest running_average_reward of -19.7 achieved!\n",
      "episode_num 203, curr_reward: -17.0, best_reward: -17.0, running_avg_reward: -19.7, curr_epsilon: 0.5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 204, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -19.71, curr_epsilon: 0.4986\n",
      "episode_num 205, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -19.72, curr_epsilon: 0.4968\n",
      "episode_num 206, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -19.75, curr_epsilon: 0.4952\n",
      "episode_num 207, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -19.72, curr_epsilon: 0.4931\n",
      "episode_num 208, curr_reward: -17.0, best_reward: -17.0, running_avg_reward: -19.7, curr_epsilon: 0.4905\n",
      "checkpointing current model weights. highest running_average_reward of -19.69 achieved!\n",
      "episode_num 209, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -19.69, curr_epsilon: 0.4884\n",
      "episode_num 210, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -19.69, curr_epsilon: 0.4865\n",
      "episode_num 211, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -19.69, curr_epsilon: 0.4848\n",
      "checkpointing current model weights. highest running_average_reward of -19.68 achieved!\n",
      "episode_num 212, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -19.68, curr_epsilon: 0.4829\n",
      "episode_num 213, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -19.69, curr_epsilon: 0.481\n",
      "episode_num 214, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -19.68, curr_epsilon: 0.479\n",
      "episode_num 215, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -19.69, curr_epsilon: 0.4771\n",
      "checkpointing current model weights. highest running_average_reward of -19.67 achieved!\n",
      "episode_num 216, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -19.67, curr_epsilon: 0.4753\n",
      "checkpointing current model weights. highest running_average_reward of -19.66 achieved!\n",
      "episode_num 217, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -19.66, curr_epsilon: 0.4735\n",
      "episode_num 218, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -19.67, curr_epsilon: 0.4716\n",
      "checkpointing current model weights. highest running_average_reward of -19.64 achieved!\n",
      "episode_num 219, curr_reward: -18.0, best_reward: -17.0, running_avg_reward: -19.64, curr_epsilon: 0.4692\n",
      "checkpointing current model weights. highest running_average_reward of -19.63 achieved!\n",
      "episode_num 220, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -19.63, curr_epsilon: 0.4672\n",
      "checkpointing current model weights. highest running_average_reward of -19.62 achieved!\n",
      "episode_num 221, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -19.62, curr_epsilon: 0.4654\n",
      "checkpointing current model weights. highest running_average_reward of -19.61 achieved!\n",
      "episode_num 222, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -19.61, curr_epsilon: 0.4635\n",
      "checkpointing current model weights. highest running_average_reward of -19.6 achieved!\n",
      "episode_num 223, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -19.6, curr_epsilon: 0.4617\n",
      "episode_num 224, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -19.6, curr_epsilon: 0.4599\n",
      "episode_num 225, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -19.6, curr_epsilon: 0.4583\n",
      "episode_num 226, curr_reward: -21.0, best_reward: -17.0, running_avg_reward: -19.6, curr_epsilon: 0.4565\n",
      "checkpointing current model weights. highest running_average_reward of -19.56 achieved!\n",
      "episode_num 227, curr_reward: -17.0, best_reward: -17.0, running_avg_reward: -19.56, curr_epsilon: 0.4545\n",
      "checkpointing current model weights. highest running_average_reward of -19.55 achieved!\n",
      "episode_num 228, curr_reward: -19.0, best_reward: -17.0, running_avg_reward: -19.55, curr_epsilon: 0.4521\n",
      "episode_num 229, curr_reward: -20.0, best_reward: -17.0, running_avg_reward: -19.56, curr_epsilon: 0.4497\n",
      "checkpointing current model weights. highest running_average_reward of -19.52 achieved!\n",
      "episode_num 230, curr_reward: -16.0, best_reward: -16.0, running_avg_reward: -19.52, curr_epsilon: 0.4469\n",
      "episode_num 231, curr_reward: -20.0, best_reward: -16.0, running_avg_reward: -19.53, curr_epsilon: 0.445\n",
      "checkpointing current model weights. highest running_average_reward of -19.51 achieved!\n",
      "episode_num 232, curr_reward: -19.0, best_reward: -16.0, running_avg_reward: -19.51, curr_epsilon: 0.4434\n",
      "checkpointing current model weights. highest running_average_reward of -19.49 achieved!\n",
      "episode_num 233, curr_reward: -19.0, best_reward: -16.0, running_avg_reward: -19.49, curr_epsilon: 0.4414\n",
      "checkpointing current model weights. highest running_average_reward of -19.47 achieved!\n",
      "episode_num 234, curr_reward: -19.0, best_reward: -16.0, running_avg_reward: -19.47, curr_epsilon: 0.4393\n",
      "checkpointing current model weights. highest running_average_reward of -19.44 achieved!\n",
      "episode_num 235, curr_reward: -18.0, best_reward: -16.0, running_avg_reward: -19.44, curr_epsilon: 0.4372\n",
      "episode_num 236, curr_reward: -21.0, best_reward: -16.0, running_avg_reward: -19.44, curr_epsilon: 0.4357\n",
      "checkpointing current model weights. highest running_average_reward of -19.42 achieved!\n",
      "episode_num 237, curr_reward: -18.0, best_reward: -16.0, running_avg_reward: -19.42, curr_epsilon: 0.4337\n",
      "episode_num 238, curr_reward: -18.0, best_reward: -16.0, running_avg_reward: -19.42, curr_epsilon: 0.4314\n",
      "checkpointing current model weights. highest running_average_reward of -19.41 achieved!\n",
      "episode_num 239, curr_reward: -18.0, best_reward: -16.0, running_avg_reward: -19.41, curr_epsilon: 0.4293\n",
      "episode_num 240, curr_reward: -19.0, best_reward: -16.0, running_avg_reward: -19.42, curr_epsilon: 0.4277\n",
      "episode_num 241, curr_reward: -19.0, best_reward: -16.0, running_avg_reward: -19.44, curr_epsilon: 0.4257\n",
      "episode_num 242, curr_reward: -19.0, best_reward: -16.0, running_avg_reward: -19.45, curr_epsilon: 0.4236\n",
      "episode_num 243, curr_reward: -17.0, best_reward: -16.0, running_avg_reward: -19.41, curr_epsilon: 0.4213\n",
      "checkpointing current model weights. highest running_average_reward of -19.39 achieved!\n",
      "episode_num 244, curr_reward: -19.0, best_reward: -16.0, running_avg_reward: -19.39, curr_epsilon: 0.4196\n",
      "episode_num 245, curr_reward: -20.0, best_reward: -16.0, running_avg_reward: -19.39, curr_epsilon: 0.418\n",
      "episode_num 246, curr_reward: -20.0, best_reward: -16.0, running_avg_reward: -19.39, curr_epsilon: 0.4161\n",
      "episode_num 247, curr_reward: -18.0, best_reward: -16.0, running_avg_reward: -19.39, curr_epsilon: 0.414\n",
      "episode_num 248, curr_reward: -21.0, best_reward: -16.0, running_avg_reward: -19.39, curr_epsilon: 0.4123\n",
      "episode_num 249, curr_reward: -19.0, best_reward: -16.0, running_avg_reward: -19.39, curr_epsilon: 0.4103\n",
      "checkpointing current model weights. highest running_average_reward of -19.36 achieved!\n",
      "episode_num 250, curr_reward: -18.0, best_reward: -16.0, running_avg_reward: -19.36, curr_epsilon: 0.408\n",
      "checkpointing current model weights. highest running_average_reward of -19.35 achieved!\n",
      "episode_num 251, curr_reward: -20.0, best_reward: -16.0, running_avg_reward: -19.35, curr_epsilon: 0.4061\n",
      "episode_num 252, curr_reward: -20.0, best_reward: -16.0, running_avg_reward: -19.35, curr_epsilon: 0.4043\n",
      "checkpointing current model weights. highest running_average_reward of -19.33 achieved!\n",
      "episode_num 253, curr_reward: -19.0, best_reward: -16.0, running_avg_reward: -19.33, curr_epsilon: 0.4022\n",
      "checkpointing current model weights. highest running_average_reward of -19.31 achieved!\n",
      "episode_num 254, curr_reward: -19.0, best_reward: -16.0, running_avg_reward: -19.31, curr_epsilon: 0.4001\n",
      "checkpointing current model weights. highest running_average_reward of -19.28 achieved!\n",
      "episode_num 255, curr_reward: -17.0, best_reward: -16.0, running_avg_reward: -19.28, curr_epsilon: 0.398\n",
      "episode_num 256, curr_reward: -17.0, best_reward: -16.0, running_avg_reward: -19.28, curr_epsilon: 0.3957\n",
      "episode_num 257, curr_reward: -20.0, best_reward: -16.0, running_avg_reward: -19.3, curr_epsilon: 0.3938\n",
      "episode_num 258, curr_reward: -19.0, best_reward: -16.0, running_avg_reward: -19.28, curr_epsilon: 0.3918\n",
      "episode_num 259, curr_reward: -18.0, best_reward: -16.0, running_avg_reward: -19.28, curr_epsilon: 0.3895\n",
      "checkpointing current model weights. highest running_average_reward of -19.27 achieved!\n",
      "episode_num 260, curr_reward: -20.0, best_reward: -16.0, running_avg_reward: -19.27, curr_epsilon: 0.388\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 261, curr_reward: -19.0, best_reward: -16.0, running_avg_reward: -19.27, curr_epsilon: 0.3859\n",
      "checkpointing current model weights. highest running_average_reward of -19.25 achieved!\n",
      "episode_num 262, curr_reward: -19.0, best_reward: -16.0, running_avg_reward: -19.25, curr_epsilon: 0.3839\n",
      "checkpointing current model weights. highest running_average_reward of -19.22 achieved!\n",
      "episode_num 263, curr_reward: -17.0, best_reward: -16.0, running_avg_reward: -19.22, curr_epsilon: 0.3817\n",
      "checkpointing current model weights. highest running_average_reward of -19.19 achieved!\n",
      "episode_num 264, curr_reward: -18.0, best_reward: -16.0, running_avg_reward: -19.19, curr_epsilon: 0.3798\n",
      "checkpointing current model weights. highest running_average_reward of -19.18 achieved!\n",
      "episode_num 265, curr_reward: -20.0, best_reward: -16.0, running_avg_reward: -19.18, curr_epsilon: 0.3781\n",
      "checkpointing current model weights. highest running_average_reward of -19.12 achieved!\n",
      "episode_num 266, curr_reward: -14.0, best_reward: -14.0, running_avg_reward: -19.12, curr_epsilon: 0.3758\n",
      "checkpointing current model weights. highest running_average_reward of -19.1 achieved!\n",
      "episode_num 267, curr_reward: -18.0, best_reward: -14.0, running_avg_reward: -19.1, curr_epsilon: 0.3742\n",
      "checkpointing current model weights. highest running_average_reward of -19.09 achieved!\n",
      "episode_num 268, curr_reward: -20.0, best_reward: -14.0, running_avg_reward: -19.09, curr_epsilon: 0.3722\n",
      "episode_num 269, curr_reward: -20.0, best_reward: -14.0, running_avg_reward: -19.09, curr_epsilon: 0.3703\n",
      "checkpointing current model weights. highest running_average_reward of -19.07 achieved!\n",
      "episode_num 270, curr_reward: -19.0, best_reward: -14.0, running_avg_reward: -19.07, curr_epsilon: 0.3686\n",
      "episode_num 271, curr_reward: -19.0, best_reward: -14.0, running_avg_reward: -19.07, curr_epsilon: 0.367\n",
      "checkpointing current model weights. highest running_average_reward of -19.05 achieved!\n",
      "episode_num 272, curr_reward: -18.0, best_reward: -14.0, running_avg_reward: -19.05, curr_epsilon: 0.365\n",
      "checkpointing current model weights. highest running_average_reward of -19.04 achieved!\n",
      "episode_num 273, curr_reward: -18.0, best_reward: -14.0, running_avg_reward: -19.04, curr_epsilon: 0.3631\n",
      "checkpointing current model weights. highest running_average_reward of -19.02 achieved!\n",
      "episode_num 274, curr_reward: -18.0, best_reward: -14.0, running_avg_reward: -19.02, curr_epsilon: 0.3616\n",
      "episode_num 275, curr_reward: -20.0, best_reward: -14.0, running_avg_reward: -19.04, curr_epsilon: 0.36\n",
      "episode_num 276, curr_reward: -19.0, best_reward: -14.0, running_avg_reward: -19.05, curr_epsilon: 0.3584\n",
      "episode_num 277, curr_reward: -18.0, best_reward: -14.0, running_avg_reward: -19.05, curr_epsilon: 0.356\n",
      "episode_num 278, curr_reward: -18.0, best_reward: -14.0, running_avg_reward: -19.04, curr_epsilon: 0.3544\n",
      "episode_num 279, curr_reward: -19.0, best_reward: -14.0, running_avg_reward: -19.02, curr_epsilon: 0.353\n",
      "episode_num 280, curr_reward: -21.0, best_reward: -14.0, running_avg_reward: -19.05, curr_epsilon: 0.3517\n",
      "episode_num 281, curr_reward: -18.0, best_reward: -14.0, running_avg_reward: -19.03, curr_epsilon: 0.35\n",
      "episode_num 282, curr_reward: -19.0, best_reward: -14.0, running_avg_reward: -19.03, curr_epsilon: 0.3481\n",
      "checkpointing current model weights. highest running_average_reward of -19.01 achieved!\n",
      "episode_num 283, curr_reward: -19.0, best_reward: -14.0, running_avg_reward: -19.01, curr_epsilon: 0.3463\n",
      "checkpointing current model weights. highest running_average_reward of -19.0 achieved!\n",
      "episode_num 284, curr_reward: -18.0, best_reward: -14.0, running_avg_reward: -19.0, curr_epsilon: 0.3446\n",
      "checkpointing current model weights. highest running_average_reward of -18.97 achieved!\n",
      "episode_num 285, curr_reward: -17.0, best_reward: -14.0, running_avg_reward: -18.97, curr_epsilon: 0.3427\n",
      "episode_num 286, curr_reward: -20.0, best_reward: -14.0, running_avg_reward: -19.0, curr_epsilon: 0.3411\n",
      "episode_num 287, curr_reward: -21.0, best_reward: -14.0, running_avg_reward: -19.04, curr_epsilon: 0.3398\n",
      "episode_num 288, curr_reward: -18.0, best_reward: -14.0, running_avg_reward: -19.04, curr_epsilon: 0.3379\n",
      "episode_num 289, curr_reward: -19.0, best_reward: -14.0, running_avg_reward: -19.04, curr_epsilon: 0.3366\n",
      "episode_num 290, curr_reward: -19.0, best_reward: -14.0, running_avg_reward: -19.05, curr_epsilon: 0.335\n",
      "episode_num 291, curr_reward: -19.0, best_reward: -14.0, running_avg_reward: -19.04, curr_epsilon: 0.3336\n",
      "episode_num 292, curr_reward: -18.0, best_reward: -14.0, running_avg_reward: -19.04, curr_epsilon: 0.3317\n",
      "episode_num 293, curr_reward: -18.0, best_reward: -14.0, running_avg_reward: -19.03, curr_epsilon: 0.3301\n",
      "episode_num 294, curr_reward: -18.0, best_reward: -14.0, running_avg_reward: -19.03, curr_epsilon: 0.3283\n",
      "episode_num 295, curr_reward: -15.0, best_reward: -14.0, running_avg_reward: -18.97, curr_epsilon: 0.3263\n",
      "checkpointing current model weights. highest running_average_reward of -18.93 achieved!\n",
      "episode_num 296, curr_reward: -17.0, best_reward: -14.0, running_avg_reward: -18.93, curr_epsilon: 0.3249\n",
      "episode_num 297, curr_reward: -20.0, best_reward: -14.0, running_avg_reward: -18.94, curr_epsilon: 0.3232\n",
      "checkpointing current model weights. highest running_average_reward of -18.92 achieved!\n",
      "episode_num 298, curr_reward: -16.0, best_reward: -14.0, running_avg_reward: -18.92, curr_epsilon: 0.3214\n",
      "checkpointing current model weights. highest running_average_reward of -18.84 achieved!\n",
      "episode_num 299, curr_reward: -13.0, best_reward: -13.0, running_avg_reward: -18.84, curr_epsilon: 0.319\n",
      "checkpointing current model weights. highest running_average_reward of -18.81 achieved!\n",
      "episode_num 300, curr_reward: -16.0, best_reward: -13.0, running_avg_reward: -18.81, curr_epsilon: 0.3165\n",
      "checkpointing current model weights. highest running_average_reward of -18.79 achieved!\n",
      "episode_num 301, curr_reward: -19.0, best_reward: -13.0, running_avg_reward: -18.79, curr_epsilon: 0.3149\n",
      "episode_num 302, curr_reward: -20.0, best_reward: -13.0, running_avg_reward: -18.79, curr_epsilon: 0.3136\n",
      "episode_num 303, curr_reward: -20.0, best_reward: -13.0, running_avg_reward: -18.82, curr_epsilon: 0.3124\n",
      "episode_num 304, curr_reward: -18.0, best_reward: -13.0, running_avg_reward: -18.79, curr_epsilon: 0.3111\n",
      "checkpointing current model weights. highest running_average_reward of -18.77 achieved!\n",
      "episode_num 305, curr_reward: -18.0, best_reward: -13.0, running_avg_reward: -18.77, curr_epsilon: 0.3094\n",
      "checkpointing current model weights. highest running_average_reward of -18.76 achieved!\n",
      "episode_num 306, curr_reward: -19.0, best_reward: -13.0, running_avg_reward: -18.76, curr_epsilon: 0.308\n",
      "episode_num 307, curr_reward: -20.0, best_reward: -13.0, running_avg_reward: -18.78, curr_epsilon: 0.3067\n",
      "episode_num 308, curr_reward: -16.0, best_reward: -13.0, running_avg_reward: -18.77, curr_epsilon: 0.3051\n",
      "episode_num 309, curr_reward: -20.0, best_reward: -13.0, running_avg_reward: -18.78, curr_epsilon: 0.3039\n",
      "episode_num 310, curr_reward: -19.0, best_reward: -13.0, running_avg_reward: -18.76, curr_epsilon: 0.3025\n",
      "episode_num 311, curr_reward: -20.0, best_reward: -13.0, running_avg_reward: -18.76, curr_epsilon: 0.3012\n",
      "checkpointing current model weights. highest running_average_reward of -18.72 achieved!\n",
      "episode_num 312, curr_reward: -16.0, best_reward: -13.0, running_avg_reward: -18.72, curr_epsilon: 0.2995\n",
      "checkpointing current model weights. highest running_average_reward of -18.67 achieved!\n",
      "episode_num 313, curr_reward: -16.0, best_reward: -13.0, running_avg_reward: -18.67, curr_epsilon: 0.2979\n",
      "checkpointing current model weights. highest running_average_reward of -18.64 achieved!\n",
      "episode_num 314, curr_reward: -16.0, best_reward: -13.0, running_avg_reward: -18.64, curr_epsilon: 0.2961\n",
      "episode_num 315, curr_reward: -20.0, best_reward: -13.0, running_avg_reward: -18.64, curr_epsilon: 0.2948\n",
      "checkpointing current model weights. highest running_average_reward of -18.6 achieved!\n",
      "episode_num 316, curr_reward: -15.0, best_reward: -13.0, running_avg_reward: -18.6, curr_epsilon: 0.293\n",
      "episode_num 317, curr_reward: -20.0, best_reward: -13.0, running_avg_reward: -18.6, curr_epsilon: 0.2914\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpointing current model weights. highest running_average_reward of -18.59 achieved!\n",
      "episode_num 318, curr_reward: -19.0, best_reward: -13.0, running_avg_reward: -18.59, curr_epsilon: 0.2896\n",
      "checkpointing current model weights. highest running_average_reward of -18.57 achieved!\n",
      "episode_num 319, curr_reward: -16.0, best_reward: -13.0, running_avg_reward: -18.57, curr_epsilon: 0.2877\n",
      "checkpointing current model weights. highest running_average_reward of -18.55 achieved!\n",
      "episode_num 320, curr_reward: -17.0, best_reward: -13.0, running_avg_reward: -18.55, curr_epsilon: 0.2861\n",
      "checkpointing current model weights. highest running_average_reward of -18.54 achieved!\n",
      "episode_num 321, curr_reward: -19.0, best_reward: -13.0, running_avg_reward: -18.54, curr_epsilon: 0.2847\n",
      "checkpointing current model weights. highest running_average_reward of -18.52 achieved!\n",
      "episode_num 322, curr_reward: -18.0, best_reward: -13.0, running_avg_reward: -18.52, curr_epsilon: 0.2831\n",
      "checkpointing current model weights. highest running_average_reward of -18.51 achieved!\n",
      "episode_num 323, curr_reward: -18.0, best_reward: -13.0, running_avg_reward: -18.51, curr_epsilon: 0.2817\n",
      "checkpointing current model weights. highest running_average_reward of -18.49 achieved!\n",
      "episode_num 324, curr_reward: -18.0, best_reward: -13.0, running_avg_reward: -18.49, curr_epsilon: 0.2801\n",
      "checkpointing current model weights. highest running_average_reward of -18.45 achieved!\n",
      "episode_num 325, curr_reward: -17.0, best_reward: -13.0, running_avg_reward: -18.45, curr_epsilon: 0.2785\n",
      "checkpointing current model weights. highest running_average_reward of -18.4 achieved!\n",
      "episode_num 326, curr_reward: -16.0, best_reward: -13.0, running_avg_reward: -18.4, curr_epsilon: 0.277\n",
      "episode_num 327, curr_reward: -19.0, best_reward: -13.0, running_avg_reward: -18.42, curr_epsilon: 0.2758\n",
      "episode_num 328, curr_reward: -20.0, best_reward: -13.0, running_avg_reward: -18.43, curr_epsilon: 0.2747\n",
      "checkpointing current model weights. highest running_average_reward of -18.38 achieved!\n",
      "episode_num 329, curr_reward: -15.0, best_reward: -13.0, running_avg_reward: -18.38, curr_epsilon: 0.273\n",
      "episode_num 330, curr_reward: -20.0, best_reward: -13.0, running_avg_reward: -18.42, curr_epsilon: 0.272\n",
      "episode_num 331, curr_reward: -18.0, best_reward: -13.0, running_avg_reward: -18.4, curr_epsilon: 0.2706\n",
      "episode_num 332, curr_reward: -20.0, best_reward: -13.0, running_avg_reward: -18.41, curr_epsilon: 0.2694\n",
      "episode_num 333, curr_reward: -18.0, best_reward: -13.0, running_avg_reward: -18.4, curr_epsilon: 0.2679\n",
      "episode_num 334, curr_reward: -18.0, best_reward: -13.0, running_avg_reward: -18.39, curr_epsilon: 0.2666\n",
      "episode_num 335, curr_reward: -19.0, best_reward: -13.0, running_avg_reward: -18.4, curr_epsilon: 0.2653\n",
      "episode_num 336, curr_reward: -19.0, best_reward: -13.0, running_avg_reward: -18.38, curr_epsilon: 0.2637\n",
      "episode_num 337, curr_reward: -18.0, best_reward: -13.0, running_avg_reward: -18.38, curr_epsilon: 0.2624\n",
      "checkpointing current model weights. highest running_average_reward of -18.36 achieved!\n",
      "episode_num 338, curr_reward: -16.0, best_reward: -13.0, running_avg_reward: -18.36, curr_epsilon: 0.2609\n",
      "episode_num 339, curr_reward: -20.0, best_reward: -13.0, running_avg_reward: -18.38, curr_epsilon: 0.2598\n",
      "episode_num 340, curr_reward: -19.0, best_reward: -13.0, running_avg_reward: -18.38, curr_epsilon: 0.2586\n",
      "episode_num 341, curr_reward: -20.0, best_reward: -13.0, running_avg_reward: -18.39, curr_epsilon: 0.2572\n",
      "episode_num 342, curr_reward: -17.0, best_reward: -13.0, running_avg_reward: -18.37, curr_epsilon: 0.2558\n",
      "episode_num 343, curr_reward: -16.0, best_reward: -13.0, running_avg_reward: -18.36, curr_epsilon: 0.2544\n",
      "episode_num 344, curr_reward: -21.0, best_reward: -13.0, running_avg_reward: -18.38, curr_epsilon: 0.2533\n",
      "episode_num 345, curr_reward: -18.0, best_reward: -13.0, running_avg_reward: -18.36, curr_epsilon: 0.2519\n",
      "checkpointing current model weights. highest running_average_reward of -18.33 achieved!\n",
      "episode_num 346, curr_reward: -17.0, best_reward: -13.0, running_avg_reward: -18.33, curr_epsilon: 0.2502\n",
      "checkpointing current model weights. highest running_average_reward of -18.32 achieved!\n",
      "episode_num 347, curr_reward: -17.0, best_reward: -13.0, running_avg_reward: -18.32, curr_epsilon: 0.2487\n",
      "checkpointing current model weights. highest running_average_reward of -18.3 achieved!\n",
      "episode_num 348, curr_reward: -19.0, best_reward: -13.0, running_avg_reward: -18.3, curr_epsilon: 0.2476\n",
      "checkpointing current model weights. highest running_average_reward of -18.29 achieved!\n",
      "episode_num 349, curr_reward: -18.0, best_reward: -13.0, running_avg_reward: -18.29, curr_epsilon: 0.2462\n",
      "checkpointing current model weights. highest running_average_reward of -18.28 achieved!\n",
      "episode_num 350, curr_reward: -17.0, best_reward: -13.0, running_avg_reward: -18.28, curr_epsilon: 0.2448\n",
      "checkpointing current model weights. highest running_average_reward of -18.27 achieved!\n",
      "episode_num 351, curr_reward: -19.0, best_reward: -13.0, running_avg_reward: -18.27, curr_epsilon: 0.2437\n",
      "checkpointing current model weights. highest running_average_reward of -18.25 achieved!\n",
      "episode_num 352, curr_reward: -18.0, best_reward: -13.0, running_avg_reward: -18.25, curr_epsilon: 0.2425\n",
      "episode_num 353, curr_reward: -20.0, best_reward: -13.0, running_avg_reward: -18.26, curr_epsilon: 0.2413\n",
      "checkpointing current model weights. highest running_average_reward of -18.24 achieved!\n",
      "episode_num 354, curr_reward: -17.0, best_reward: -13.0, running_avg_reward: -18.24, curr_epsilon: 0.2401\n",
      "episode_num 355, curr_reward: -19.0, best_reward: -13.0, running_avg_reward: -18.26, curr_epsilon: 0.239\n",
      "episode_num 356, curr_reward: -21.0, best_reward: -13.0, running_avg_reward: -18.3, curr_epsilon: 0.2381\n",
      "checkpointing current model weights. highest running_average_reward of -18.22 achieved!\n",
      "episode_num 357, curr_reward: -12.0, best_reward: -12.0, running_avg_reward: -18.22, curr_epsilon: 0.2363\n",
      "episode_num 358, curr_reward: -19.0, best_reward: -12.0, running_avg_reward: -18.22, curr_epsilon: 0.2353\n",
      "episode_num 359, curr_reward: -19.0, best_reward: -12.0, running_avg_reward: -18.23, curr_epsilon: 0.2336\n",
      "episode_num 360, curr_reward: -19.0, best_reward: -12.0, running_avg_reward: -18.22, curr_epsilon: 0.2324\n",
      "checkpointing current model weights. highest running_average_reward of -18.16 achieved!\n",
      "episode_num 361, curr_reward: -13.0, best_reward: -12.0, running_avg_reward: -18.16, curr_epsilon: 0.2307\n",
      "episode_num 362, curr_reward: -19.0, best_reward: -12.0, running_avg_reward: -18.16, curr_epsilon: 0.2296\n",
      "episode_num 363, curr_reward: -21.0, best_reward: -12.0, running_avg_reward: -18.2, curr_epsilon: 0.2287\n",
      "episode_num 364, curr_reward: -17.0, best_reward: -12.0, running_avg_reward: -18.19, curr_epsilon: 0.2272\n",
      "episode_num 365, curr_reward: -18.0, best_reward: -12.0, running_avg_reward: -18.17, curr_epsilon: 0.2261\n",
      "episode_num 366, curr_reward: -16.0, best_reward: -12.0, running_avg_reward: -18.19, curr_epsilon: 0.2248\n",
      "episode_num 367, curr_reward: -19.0, best_reward: -12.0, running_avg_reward: -18.2, curr_epsilon: 0.2237\n",
      "episode_num 368, curr_reward: -18.0, best_reward: -12.0, running_avg_reward: -18.18, curr_epsilon: 0.2224\n",
      "episode_num 369, curr_reward: -19.0, best_reward: -12.0, running_avg_reward: -18.17, curr_epsilon: 0.2213\n",
      "episode_num 370, curr_reward: -18.0, best_reward: -12.0, running_avg_reward: -18.16, curr_epsilon: 0.22\n",
      "episode_num 371, curr_reward: -21.0, best_reward: -12.0, running_avg_reward: -18.18, curr_epsilon: 0.2191\n",
      "episode_num 372, curr_reward: -20.0, best_reward: -12.0, running_avg_reward: -18.2, curr_epsilon: 0.2181\n",
      "episode_num 373, curr_reward: -19.0, best_reward: -12.0, running_avg_reward: -18.21, curr_epsilon: 0.2166\n",
      "episode_num 374, curr_reward: -19.0, best_reward: -12.0, running_avg_reward: -18.22, curr_epsilon: 0.2152\n",
      "episode_num 375, curr_reward: -17.0, best_reward: -12.0, running_avg_reward: -18.19, curr_epsilon: 0.2141\n",
      "episode_num 376, curr_reward: -20.0, best_reward: -12.0, running_avg_reward: -18.2, curr_epsilon: 0.213\n",
      "episode_num 377, curr_reward: -19.0, best_reward: -12.0, running_avg_reward: -18.21, curr_epsilon: 0.2121\n",
      "episode_num 378, curr_reward: -17.0, best_reward: -12.0, running_avg_reward: -18.2, curr_epsilon: 0.2109\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 379, curr_reward: -17.0, best_reward: -12.0, running_avg_reward: -18.18, curr_epsilon: 0.2095\n",
      "checkpointing current model weights. highest running_average_reward of -18.14 achieved!\n",
      "episode_num 380, curr_reward: -17.0, best_reward: -12.0, running_avg_reward: -18.14, curr_epsilon: 0.2082\n",
      "checkpointing current model weights. highest running_average_reward of -18.12 achieved!\n",
      "episode_num 381, curr_reward: -16.0, best_reward: -12.0, running_avg_reward: -18.12, curr_epsilon: 0.2068\n",
      "episode_num 382, curr_reward: -19.0, best_reward: -12.0, running_avg_reward: -18.12, curr_epsilon: 0.2059\n",
      "checkpointing current model weights. highest running_average_reward of -18.1 achieved!\n",
      "episode_num 383, curr_reward: -17.0, best_reward: -12.0, running_avg_reward: -18.1, curr_epsilon: 0.2046\n",
      "checkpointing current model weights. highest running_average_reward of -18.08 achieved!\n",
      "episode_num 384, curr_reward: -16.0, best_reward: -12.0, running_avg_reward: -18.08, curr_epsilon: 0.2036\n",
      "checkpointing current model weights. highest running_average_reward of -18.02 achieved!\n",
      "episode_num 385, curr_reward: -11.0, best_reward: -11.0, running_avg_reward: -18.02, curr_epsilon: 0.2021\n",
      "checkpointing current model weights. highest running_average_reward of -18.0 achieved!\n",
      "episode_num 386, curr_reward: -18.0, best_reward: -11.0, running_avg_reward: -18.0, curr_epsilon: 0.201\n",
      "checkpointing current model weights. highest running_average_reward of -17.97 achieved!\n",
      "episode_num 387, curr_reward: -18.0, best_reward: -11.0, running_avg_reward: -17.97, curr_epsilon: 0.1999\n",
      "checkpointing current model weights. highest running_average_reward of -17.95 achieved!\n",
      "episode_num 388, curr_reward: -16.0, best_reward: -11.0, running_avg_reward: -17.95, curr_epsilon: 0.1988\n",
      "checkpointing current model weights. highest running_average_reward of -17.94 achieved!\n",
      "episode_num 389, curr_reward: -18.0, best_reward: -11.0, running_avg_reward: -17.94, curr_epsilon: 0.1979\n",
      "checkpointing current model weights. highest running_average_reward of -17.92 achieved!\n",
      "episode_num 390, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.92, curr_epsilon: 0.1967\n",
      "checkpointing current model weights. highest running_average_reward of -17.9 achieved!\n",
      "episode_num 391, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.9, curr_epsilon: 0.1956\n",
      "checkpointing current model weights. highest running_average_reward of -17.88 achieved!\n",
      "episode_num 392, curr_reward: -16.0, best_reward: -11.0, running_avg_reward: -17.88, curr_epsilon: 0.1944\n",
      "checkpointing current model weights. highest running_average_reward of -17.87 achieved!\n",
      "episode_num 393, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.87, curr_epsilon: 0.1932\n",
      "checkpointing current model weights. highest running_average_reward of -17.84 achieved!\n",
      "episode_num 394, curr_reward: -15.0, best_reward: -11.0, running_avg_reward: -17.84, curr_epsilon: 0.1918\n",
      "episode_num 395, curr_reward: -18.0, best_reward: -11.0, running_avg_reward: -17.87, curr_epsilon: 0.1908\n",
      "episode_num 396, curr_reward: -18.0, best_reward: -11.0, running_avg_reward: -17.88, curr_epsilon: 0.19\n",
      "episode_num 397, curr_reward: -18.0, best_reward: -11.0, running_avg_reward: -17.86, curr_epsilon: 0.189\n",
      "episode_num 398, curr_reward: -14.0, best_reward: -11.0, running_avg_reward: -17.84, curr_epsilon: 0.1878\n",
      "episode_num 399, curr_reward: -16.0, best_reward: -11.0, running_avg_reward: -17.87, curr_epsilon: 0.1868\n",
      "episode_num 400, curr_reward: -18.0, best_reward: -11.0, running_avg_reward: -17.89, curr_epsilon: 0.1857\n",
      "episode_num 401, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -17.89, curr_epsilon: 0.1848\n",
      "episode_num 402, curr_reward: -20.0, best_reward: -11.0, running_avg_reward: -17.89, curr_epsilon: 0.1841\n",
      "episode_num 403, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -17.88, curr_epsilon: 0.1831\n",
      "episode_num 404, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.87, curr_epsilon: 0.1822\n",
      "episode_num 405, curr_reward: -20.0, best_reward: -11.0, running_avg_reward: -17.89, curr_epsilon: 0.1815\n",
      "episode_num 406, curr_reward: -20.0, best_reward: -11.0, running_avg_reward: -17.9, curr_epsilon: 0.1805\n",
      "episode_num 407, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.87, curr_epsilon: 0.1794\n",
      "episode_num 408, curr_reward: -18.0, best_reward: -11.0, running_avg_reward: -17.89, curr_epsilon: 0.1782\n",
      "episode_num 409, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -17.88, curr_epsilon: 0.1773\n",
      "episode_num 410, curr_reward: -18.0, best_reward: -11.0, running_avg_reward: -17.87, curr_epsilon: 0.1763\n",
      "episode_num 411, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.84, curr_epsilon: 0.1753\n",
      "episode_num 412, curr_reward: -20.0, best_reward: -11.0, running_avg_reward: -17.88, curr_epsilon: 0.1745\n",
      "episode_num 413, curr_reward: -15.0, best_reward: -11.0, running_avg_reward: -17.87, curr_epsilon: 0.1732\n",
      "episode_num 414, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -17.9, curr_epsilon: 0.1719\n",
      "episode_num 415, curr_reward: -18.0, best_reward: -11.0, running_avg_reward: -17.88, curr_epsilon: 0.1708\n",
      "episode_num 416, curr_reward: -20.0, best_reward: -11.0, running_avg_reward: -17.93, curr_epsilon: 0.17\n",
      "episode_num 417, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.9, curr_epsilon: 0.169\n",
      "episode_num 418, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -17.9, curr_epsilon: 0.1682\n",
      "episode_num 419, curr_reward: -20.0, best_reward: -11.0, running_avg_reward: -17.94, curr_epsilon: 0.1673\n",
      "episode_num 420, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -17.96, curr_epsilon: 0.1664\n",
      "episode_num 421, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.94, curr_epsilon: 0.1654\n",
      "episode_num 422, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -17.95, curr_epsilon: 0.1646\n",
      "episode_num 423, curr_reward: -20.0, best_reward: -11.0, running_avg_reward: -17.97, curr_epsilon: 0.1639\n",
      "episode_num 424, curr_reward: -18.0, best_reward: -11.0, running_avg_reward: -17.97, curr_epsilon: 0.1632\n",
      "episode_num 425, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.97, curr_epsilon: 0.1623\n",
      "episode_num 426, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.98, curr_epsilon: 0.1613\n",
      "episode_num 427, curr_reward: -16.0, best_reward: -11.0, running_avg_reward: -17.95, curr_epsilon: 0.16\n",
      "episode_num 428, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.92, curr_epsilon: 0.1589\n",
      "episode_num 429, curr_reward: -14.0, best_reward: -11.0, running_avg_reward: -17.91, curr_epsilon: 0.1577\n",
      "episode_num 430, curr_reward: -18.0, best_reward: -11.0, running_avg_reward: -17.89, curr_epsilon: 0.1568\n",
      "episode_num 431, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -17.9, curr_epsilon: 0.1559\n",
      "episode_num 432, curr_reward: -15.0, best_reward: -11.0, running_avg_reward: -17.85, curr_epsilon: 0.1548\n",
      "checkpointing current model weights. highest running_average_reward of -17.81 achieved!\n",
      "episode_num 433, curr_reward: -14.0, best_reward: -11.0, running_avg_reward: -17.81, curr_epsilon: 0.1537\n",
      "episode_num 434, curr_reward: -20.0, best_reward: -11.0, running_avg_reward: -17.83, curr_epsilon: 0.1531\n",
      "episode_num 435, curr_reward: -20.0, best_reward: -11.0, running_avg_reward: -17.84, curr_epsilon: 0.1521\n",
      "episode_num 436, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -17.84, curr_epsilon: 0.1513\n",
      "episode_num 437, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.83, curr_epsilon: 0.1504\n",
      "episode_num 438, curr_reward: -16.0, best_reward: -11.0, running_avg_reward: -17.83, curr_epsilon: 0.1492\n",
      "episode_num 439, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -17.82, curr_epsilon: 0.1484\n",
      "episode_num 440, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -17.82, curr_epsilon: 0.1475\n",
      "checkpointing current model weights. highest running_average_reward of -17.79 achieved!\n",
      "episode_num 441, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.79, curr_epsilon: 0.1467\n",
      "checkpointing current model weights. highest running_average_reward of -17.78 achieved!\n",
      "episode_num 442, curr_reward: -16.0, best_reward: -11.0, running_avg_reward: -17.78, curr_epsilon: 0.1456\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 443, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -17.81, curr_epsilon: 0.1448\n",
      "checkpointing current model weights. highest running_average_reward of -17.76 achieved!\n",
      "episode_num 444, curr_reward: -16.0, best_reward: -11.0, running_avg_reward: -17.76, curr_epsilon: 0.1438\n",
      "checkpointing current model weights. highest running_average_reward of -17.71 achieved!\n",
      "episode_num 445, curr_reward: -13.0, best_reward: -11.0, running_avg_reward: -17.71, curr_epsilon: 0.1426\n",
      "checkpointing current model weights. highest running_average_reward of -17.69 achieved!\n",
      "episode_num 446, curr_reward: -15.0, best_reward: -11.0, running_avg_reward: -17.69, curr_epsilon: 0.1416\n",
      "checkpointing current model weights. highest running_average_reward of -17.68 achieved!\n",
      "episode_num 447, curr_reward: -16.0, best_reward: -11.0, running_avg_reward: -17.68, curr_epsilon: 0.1407\n",
      "checkpointing current model weights. highest running_average_reward of -17.66 achieved!\n",
      "episode_num 448, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.66, curr_epsilon: 0.1398\n",
      "checkpointing current model weights. highest running_average_reward of -17.64 achieved!\n",
      "episode_num 449, curr_reward: -16.0, best_reward: -11.0, running_avg_reward: -17.64, curr_epsilon: 0.139\n",
      "episode_num 450, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -17.66, curr_epsilon: 0.138\n",
      "checkpointing current model weights. highest running_average_reward of -17.6 achieved!\n",
      "episode_num 451, curr_reward: -13.0, best_reward: -11.0, running_avg_reward: -17.6, curr_epsilon: 0.1368\n",
      "checkpointing current model weights. highest running_average_reward of -17.57 achieved!\n",
      "episode_num 452, curr_reward: -15.0, best_reward: -11.0, running_avg_reward: -17.57, curr_epsilon: 0.1359\n",
      "checkpointing current model weights. highest running_average_reward of -17.52 achieved!\n",
      "episode_num 453, curr_reward: -15.0, best_reward: -11.0, running_avg_reward: -17.52, curr_epsilon: 0.1348\n",
      "checkpointing current model weights. highest running_average_reward of -17.48 achieved!\n",
      "episode_num 454, curr_reward: -13.0, best_reward: -11.0, running_avg_reward: -17.48, curr_epsilon: 0.1337\n",
      "checkpointing current model weights. highest running_average_reward of -17.42 achieved!\n",
      "episode_num 455, curr_reward: -13.0, best_reward: -11.0, running_avg_reward: -17.42, curr_epsilon: 0.1327\n",
      "checkpointing current model weights. highest running_average_reward of -17.37 achieved!\n",
      "episode_num 456, curr_reward: -16.0, best_reward: -11.0, running_avg_reward: -17.37, curr_epsilon: 0.132\n",
      "episode_num 457, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -17.44, curr_epsilon: 0.1313\n",
      "episode_num 458, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.42, curr_epsilon: 0.1304\n",
      "episode_num 459, curr_reward: -18.0, best_reward: -11.0, running_avg_reward: -17.41, curr_epsilon: 0.1296\n",
      "episode_num 460, curr_reward: -15.0, best_reward: -11.0, running_avg_reward: -17.37, curr_epsilon: 0.1288\n",
      "episode_num 461, curr_reward: -16.0, best_reward: -11.0, running_avg_reward: -17.4, curr_epsilon: 0.1279\n",
      "checkpointing current model weights. highest running_average_reward of -17.35 achieved!\n",
      "episode_num 462, curr_reward: -14.0, best_reward: -11.0, running_avg_reward: -17.35, curr_epsilon: 0.127\n",
      "checkpointing current model weights. highest running_average_reward of -17.27 achieved!\n",
      "episode_num 463, curr_reward: -13.0, best_reward: -11.0, running_avg_reward: -17.27, curr_epsilon: 0.1261\n",
      "checkpointing current model weights. highest running_average_reward of -17.25 achieved!\n",
      "episode_num 464, curr_reward: -15.0, best_reward: -11.0, running_avg_reward: -17.25, curr_epsilon: 0.1251\n",
      "checkpointing current model weights. highest running_average_reward of -17.23 achieved!\n",
      "episode_num 465, curr_reward: -16.0, best_reward: -11.0, running_avg_reward: -17.23, curr_epsilon: 0.1243\n",
      "checkpointing current model weights. highest running_average_reward of -17.19 achieved!\n",
      "episode_num 466, curr_reward: -12.0, best_reward: -11.0, running_avg_reward: -17.19, curr_epsilon: 0.1232\n",
      "checkpointing current model weights. highest running_average_reward of -17.16 achieved!\n",
      "episode_num 467, curr_reward: -16.0, best_reward: -11.0, running_avg_reward: -17.16, curr_epsilon: 0.1224\n",
      "episode_num 468, curr_reward: -18.0, best_reward: -11.0, running_avg_reward: -17.16, curr_epsilon: 0.1215\n",
      "checkpointing current model weights. highest running_average_reward of -17.11 achieved!\n",
      "episode_num 469, curr_reward: -14.0, best_reward: -11.0, running_avg_reward: -17.11, curr_epsilon: 0.1207\n",
      "episode_num 470, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -17.12, curr_epsilon: 0.12\n",
      "checkpointing current model weights. highest running_average_reward of -17.09 achieved!\n",
      "episode_num 471, curr_reward: -18.0, best_reward: -11.0, running_avg_reward: -17.09, curr_epsilon: 0.1193\n",
      "episode_num 472, curr_reward: -21.0, best_reward: -11.0, running_avg_reward: -17.1, curr_epsilon: 0.1185\n",
      "checkpointing current model weights. highest running_average_reward of -17.05 achieved!\n",
      "episode_num 473, curr_reward: -14.0, best_reward: -11.0, running_avg_reward: -17.05, curr_epsilon: 0.1176\n",
      "checkpointing current model weights. highest running_average_reward of -17.01 achieved!\n",
      "episode_num 474, curr_reward: -15.0, best_reward: -11.0, running_avg_reward: -17.01, curr_epsilon: 0.1167\n",
      "episode_num 475, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -17.01, curr_epsilon: 0.116\n",
      "checkpointing current model weights. highest running_average_reward of -16.94 achieved!\n",
      "episode_num 476, curr_reward: -13.0, best_reward: -11.0, running_avg_reward: -16.94, curr_epsilon: 0.115\n",
      "checkpointing current model weights. highest running_average_reward of -16.92 achieved!\n",
      "episode_num 477, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -16.92, curr_epsilon: 0.1143\n",
      "episode_num 478, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -16.94, curr_epsilon: 0.1136\n",
      "episode_num 479, curr_reward: -20.0, best_reward: -11.0, running_avg_reward: -16.97, curr_epsilon: 0.1131\n",
      "episode_num 480, curr_reward: -14.0, best_reward: -11.0, running_avg_reward: -16.94, curr_epsilon: 0.1123\n",
      "checkpointing current model weights. highest running_average_reward of -16.9 achieved!\n",
      "episode_num 481, curr_reward: -12.0, best_reward: -11.0, running_avg_reward: -16.9, curr_epsilon: 0.1113\n",
      "episode_num 482, curr_reward: -19.0, best_reward: -11.0, running_avg_reward: -16.9, curr_epsilon: 0.1105\n",
      "episode_num 483, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -16.9, curr_epsilon: 0.1096\n",
      "checkpointing current model weights. highest running_average_reward of -16.89 achieved!\n",
      "episode_num 484, curr_reward: -15.0, best_reward: -11.0, running_avg_reward: -16.89, curr_epsilon: 0.1086\n",
      "episode_num 485, curr_reward: -17.0, best_reward: -11.0, running_avg_reward: -16.95, curr_epsilon: 0.1079\n",
      "checkpointing current model weights. highest running_average_reward of -16.85 achieved!\n",
      "episode_num 486, curr_reward: -8.0, best_reward: -8.0, running_avg_reward: -16.85, curr_epsilon: 0.1068\n",
      "checkpointing current model weights. highest running_average_reward of -16.82 achieved!\n",
      "episode_num 487, curr_reward: -15.0, best_reward: -8.0, running_avg_reward: -16.82, curr_epsilon: 0.1061\n",
      "checkpointing current model weights. highest running_average_reward of -16.8 achieved!\n",
      "episode_num 488, curr_reward: -14.0, best_reward: -8.0, running_avg_reward: -16.8, curr_epsilon: 0.1052\n",
      "checkpointing current model weights. highest running_average_reward of -16.78 achieved!\n",
      "episode_num 489, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -16.78, curr_epsilon: 0.1045\n",
      "episode_num 490, curr_reward: -21.0, best_reward: -8.0, running_avg_reward: -16.82, curr_epsilon: 0.104\n",
      "episode_num 491, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -16.81, curr_epsilon: 0.1034\n",
      "episode_num 492, curr_reward: -13.0, best_reward: -8.0, running_avg_reward: -16.78, curr_epsilon: 0.1025\n",
      "checkpointing current model weights. highest running_average_reward of -16.74 achieved!\n",
      "episode_num 493, curr_reward: -13.0, best_reward: -8.0, running_avg_reward: -16.74, curr_epsilon: 0.1015\n",
      "episode_num 494, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -16.76, curr_epsilon: 0.1008\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 495, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -16.75, curr_epsilon: 0.1001\n",
      "checkpointing current model weights. highest running_average_reward of -16.73 achieved!\n",
      "episode_num 496, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -16.73, curr_epsilon: 0.0995\n",
      "checkpointing current model weights. highest running_average_reward of -16.7 achieved!\n",
      "episode_num 497, curr_reward: -15.0, best_reward: -8.0, running_avg_reward: -16.7, curr_epsilon: 0.0986\n",
      "episode_num 498, curr_reward: -14.0, best_reward: -8.0, running_avg_reward: -16.7, curr_epsilon: 0.0979\n",
      "checkpointing current model weights. highest running_average_reward of -16.69 achieved!\n",
      "episode_num 499, curr_reward: -15.0, best_reward: -8.0, running_avg_reward: -16.69, curr_epsilon: 0.0971\n",
      "checkpointing current model weights. highest running_average_reward of -16.65 achieved!\n",
      "episode_num 500, curr_reward: -14.0, best_reward: -8.0, running_avg_reward: -16.65, curr_epsilon: 0.0963\n",
      "checkpointing current model weights. highest running_average_reward of -16.63 achieved!\n",
      "episode_num 501, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -16.63, curr_epsilon: 0.0957\n",
      "checkpointing current model weights. highest running_average_reward of -16.6 achieved!\n",
      "episode_num 502, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -16.6, curr_epsilon: 0.095\n",
      "checkpointing current model weights. highest running_average_reward of -16.59 achieved!\n",
      "episode_num 503, curr_reward: -18.0, best_reward: -8.0, running_avg_reward: -16.59, curr_epsilon: 0.0943\n",
      "checkpointing current model weights. highest running_average_reward of -16.57 achieved!\n",
      "episode_num 504, curr_reward: -15.0, best_reward: -8.0, running_avg_reward: -16.57, curr_epsilon: 0.0936\n",
      "checkpointing current model weights. highest running_average_reward of -16.5 achieved!\n",
      "episode_num 505, curr_reward: -13.0, best_reward: -8.0, running_avg_reward: -16.5, curr_epsilon: 0.0928\n",
      "checkpointing current model weights. highest running_average_reward of -16.48 achieved!\n",
      "episode_num 506, curr_reward: -18.0, best_reward: -8.0, running_avg_reward: -16.48, curr_epsilon: 0.0922\n",
      "checkpointing current model weights. highest running_average_reward of -16.47 achieved!\n",
      "episode_num 507, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -16.47, curr_epsilon: 0.0914\n",
      "checkpointing current model weights. highest running_average_reward of -16.46 achieved!\n",
      "episode_num 508, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -16.46, curr_epsilon: 0.0908\n",
      "checkpointing current model weights. highest running_average_reward of -16.44 achieved!\n",
      "episode_num 509, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -16.44, curr_epsilon: 0.0902\n",
      "checkpointing current model weights. highest running_average_reward of -16.42 achieved!\n",
      "episode_num 510, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -16.42, curr_epsilon: 0.0897\n",
      "episode_num 511, curr_reward: -19.0, best_reward: -8.0, running_avg_reward: -16.44, curr_epsilon: 0.0892\n",
      "episode_num 512, curr_reward: -19.0, best_reward: -8.0, running_avg_reward: -16.43, curr_epsilon: 0.0886\n",
      "episode_num 513, curr_reward: -18.0, best_reward: -8.0, running_avg_reward: -16.46, curr_epsilon: 0.0881\n",
      "episode_num 514, curr_reward: -15.0, best_reward: -8.0, running_avg_reward: -16.42, curr_epsilon: 0.0874\n",
      "episode_num 515, curr_reward: -18.0, best_reward: -8.0, running_avg_reward: -16.42, curr_epsilon: 0.0869\n",
      "checkpointing current model weights. highest running_average_reward of -16.4 achieved!\n",
      "episode_num 516, curr_reward: -18.0, best_reward: -8.0, running_avg_reward: -16.4, curr_epsilon: 0.0863\n",
      "checkpointing current model weights. highest running_average_reward of -16.38 achieved!\n",
      "episode_num 517, curr_reward: -15.0, best_reward: -8.0, running_avg_reward: -16.38, curr_epsilon: 0.0856\n",
      "checkpointing current model weights. highest running_average_reward of -16.33 achieved!\n",
      "episode_num 518, curr_reward: -14.0, best_reward: -8.0, running_avg_reward: -16.33, curr_epsilon: 0.0851\n",
      "checkpointing current model weights. highest running_average_reward of -16.3 achieved!\n",
      "episode_num 519, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -16.3, curr_epsilon: 0.0846\n",
      "checkpointing current model weights. highest running_average_reward of -16.27 achieved!\n",
      "episode_num 520, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -16.27, curr_epsilon: 0.084\n",
      "episode_num 521, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -16.27, curr_epsilon: 0.0835\n",
      "checkpointing current model weights. highest running_average_reward of -16.26 achieved!\n",
      "episode_num 522, curr_reward: -18.0, best_reward: -8.0, running_avg_reward: -16.26, curr_epsilon: 0.0829\n",
      "checkpointing current model weights. highest running_average_reward of -16.22 achieved!\n",
      "episode_num 523, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -16.22, curr_epsilon: 0.0823\n",
      "episode_num 524, curr_reward: -18.0, best_reward: -8.0, running_avg_reward: -16.22, curr_epsilon: 0.0817\n",
      "checkpointing current model weights. highest running_average_reward of -16.2 achieved!\n",
      "episode_num 525, curr_reward: -15.0, best_reward: -8.0, running_avg_reward: -16.2, curr_epsilon: 0.0811\n",
      "checkpointing current model weights. highest running_average_reward of -16.18 achieved!\n",
      "episode_num 526, curr_reward: -15.0, best_reward: -8.0, running_avg_reward: -16.18, curr_epsilon: 0.0806\n",
      "episode_num 527, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -16.18, curr_epsilon: 0.0799\n",
      "episode_num 528, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -16.18, curr_epsilon: 0.0792\n",
      "episode_num 529, curr_reward: -14.0, best_reward: -8.0, running_avg_reward: -16.18, curr_epsilon: 0.0786\n",
      "checkpointing current model weights. highest running_average_reward of -16.16 achieved!\n",
      "episode_num 530, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -16.16, curr_epsilon: 0.078\n",
      "checkpointing current model weights. highest running_average_reward of -16.13 achieved!\n",
      "episode_num 531, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -16.13, curr_epsilon: 0.0775\n",
      "episode_num 532, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -16.14, curr_epsilon: 0.0769\n",
      "episode_num 533, curr_reward: -15.0, best_reward: -8.0, running_avg_reward: -16.15, curr_epsilon: 0.0762\n",
      "checkpointing current model weights. highest running_average_reward of -16.1 achieved!\n",
      "episode_num 534, curr_reward: -15.0, best_reward: -8.0, running_avg_reward: -16.1, curr_epsilon: 0.0757\n",
      "checkpointing current model weights. highest running_average_reward of -16.05 achieved!\n",
      "episode_num 535, curr_reward: -15.0, best_reward: -8.0, running_avg_reward: -16.05, curr_epsilon: 0.0751\n",
      "checkpointing current model weights. highest running_average_reward of -16.02 achieved!\n",
      "episode_num 536, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -16.02, curr_epsilon: 0.0745\n",
      "episode_num 537, curr_reward: -20.0, best_reward: -8.0, running_avg_reward: -16.05, curr_epsilon: 0.0741\n",
      "episode_num 538, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -16.05, curr_epsilon: 0.0735\n",
      "episode_num 539, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -16.02, curr_epsilon: 0.0731\n",
      "checkpointing current model weights. highest running_average_reward of -16.0 achieved!\n",
      "episode_num 540, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -16.0, curr_epsilon: 0.0727\n",
      "episode_num 541, curr_reward: -19.0, best_reward: -8.0, running_avg_reward: -16.02, curr_epsilon: 0.0723\n",
      "episode_num 542, curr_reward: -19.0, best_reward: -8.0, running_avg_reward: -16.05, curr_epsilon: 0.0718\n",
      "episode_num 543, curr_reward: -21.0, best_reward: -8.0, running_avg_reward: -16.07, curr_epsilon: 0.0713\n",
      "episode_num 544, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -16.08, curr_epsilon: 0.0708\n",
      "episode_num 545, curr_reward: -13.0, best_reward: -8.0, running_avg_reward: -16.08, curr_epsilon: 0.0702\n",
      "episode_num 546, curr_reward: -15.0, best_reward: -8.0, running_avg_reward: -16.08, curr_epsilon: 0.0697\n",
      "episode_num 547, curr_reward: -8.0, best_reward: -8.0, running_avg_reward: -16.0, curr_epsilon: 0.069\n",
      "episode_num 548, curr_reward: -20.0, best_reward: -8.0, running_avg_reward: -16.03, curr_epsilon: 0.0686\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 549, curr_reward: -15.0, best_reward: -8.0, running_avg_reward: -16.02, curr_epsilon: 0.068\n",
      "checkpointing current model weights. highest running_average_reward of -15.95 achieved!\n",
      "episode_num 550, curr_reward: -12.0, best_reward: -8.0, running_avg_reward: -15.95, curr_epsilon: 0.0674\n",
      "episode_num 551, curr_reward: -14.0, best_reward: -8.0, running_avg_reward: -15.96, curr_epsilon: 0.0669\n",
      "episode_num 552, curr_reward: -20.0, best_reward: -8.0, running_avg_reward: -16.01, curr_epsilon: 0.0664\n",
      "episode_num 553, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -16.03, curr_epsilon: 0.066\n",
      "episode_num 554, curr_reward: -13.0, best_reward: -8.0, running_avg_reward: -16.03, curr_epsilon: 0.0653\n",
      "episode_num 555, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -16.07, curr_epsilon: 0.0649\n",
      "episode_num 556, curr_reward: -11.0, best_reward: -8.0, running_avg_reward: -16.02, curr_epsilon: 0.0643\n",
      "episode_num 557, curr_reward: -13.0, best_reward: -8.0, running_avg_reward: -15.96, curr_epsilon: 0.0636\n",
      "episode_num 558, curr_reward: -19.0, best_reward: -8.0, running_avg_reward: -15.98, curr_epsilon: 0.0632\n",
      "episode_num 559, curr_reward: -15.0, best_reward: -8.0, running_avg_reward: -15.95, curr_epsilon: 0.0628\n",
      "episode_num 560, curr_reward: -19.0, best_reward: -8.0, running_avg_reward: -15.99, curr_epsilon: 0.0624\n",
      "episode_num 561, curr_reward: -12.0, best_reward: -8.0, running_avg_reward: -15.95, curr_epsilon: 0.0618\n",
      "episode_num 562, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -15.98, curr_epsilon: 0.0615\n",
      "episode_num 563, curr_reward: -19.0, best_reward: -8.0, running_avg_reward: -16.04, curr_epsilon: 0.0611\n",
      "episode_num 564, curr_reward: -13.0, best_reward: -8.0, running_avg_reward: -16.02, curr_epsilon: 0.0606\n",
      "episode_num 565, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -16.03, curr_epsilon: 0.0601\n",
      "episode_num 566, curr_reward: -14.0, best_reward: -8.0, running_avg_reward: -16.05, curr_epsilon: 0.0596\n",
      "episode_num 567, curr_reward: -14.0, best_reward: -8.0, running_avg_reward: -16.03, curr_epsilon: 0.0591\n",
      "episode_num 568, curr_reward: -14.0, best_reward: -8.0, running_avg_reward: -15.99, curr_epsilon: 0.0587\n",
      "episode_num 569, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -16.01, curr_epsilon: 0.0582\n",
      "episode_num 570, curr_reward: -13.0, best_reward: -8.0, running_avg_reward: -15.95, curr_epsilon: 0.0578\n",
      "checkpointing current model weights. highest running_average_reward of -15.91 achieved!\n",
      "episode_num 571, curr_reward: -14.0, best_reward: -8.0, running_avg_reward: -15.91, curr_epsilon: 0.0573\n",
      "checkpointing current model weights. highest running_average_reward of -15.82 achieved!\n",
      "episode_num 572, curr_reward: -12.0, best_reward: -8.0, running_avg_reward: -15.82, curr_epsilon: 0.0568\n",
      "episode_num 573, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -15.84, curr_epsilon: 0.0563\n",
      "checkpointing current model weights. highest running_average_reward of -15.81 achieved!\n",
      "episode_num 574, curr_reward: -12.0, best_reward: -8.0, running_avg_reward: -15.81, curr_epsilon: 0.0558\n",
      "checkpointing current model weights. highest running_average_reward of -15.8 achieved!\n",
      "episode_num 575, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -15.8, curr_epsilon: 0.0554\n",
      "episode_num 576, curr_reward: -14.0, best_reward: -8.0, running_avg_reward: -15.81, curr_epsilon: 0.0548\n",
      "checkpointing current model weights. highest running_average_reward of -15.77 achieved!\n",
      "episode_num 577, curr_reward: -13.0, best_reward: -8.0, running_avg_reward: -15.77, curr_epsilon: 0.0544\n",
      "episode_num 578, curr_reward: -19.0, best_reward: -8.0, running_avg_reward: -15.77, curr_epsilon: 0.0541\n",
      "checkpointing current model weights. highest running_average_reward of -15.74 achieved!\n",
      "episode_num 579, curr_reward: -17.0, best_reward: -8.0, running_avg_reward: -15.74, curr_epsilon: 0.0538\n",
      "episode_num 580, curr_reward: -16.0, best_reward: -8.0, running_avg_reward: -15.76, curr_epsilon: 0.0533\n",
      "episode_num 581, curr_reward: -21.0, best_reward: -8.0, running_avg_reward: -15.85, curr_epsilon: 0.053\n",
      "checkpointing current model weights. highest running_average_reward of -15.73 achieved!\n",
      "episode_num 582, curr_reward: -7.0, best_reward: -7.0, running_avg_reward: -15.73, curr_epsilon: 0.0525\n",
      "checkpointing current model weights. highest running_average_reward of -15.7 achieved!\n",
      "episode_num 583, curr_reward: -14.0, best_reward: -7.0, running_avg_reward: -15.7, curr_epsilon: 0.052\n",
      "checkpointing current model weights. highest running_average_reward of -15.64 achieved!\n",
      "episode_num 584, curr_reward: -9.0, best_reward: -7.0, running_avg_reward: -15.64, curr_epsilon: 0.0515\n",
      "checkpointing current model weights. highest running_average_reward of -15.57 achieved!\n",
      "episode_num 585, curr_reward: -10.0, best_reward: -7.0, running_avg_reward: -15.57, curr_epsilon: 0.051\n",
      "episode_num 586, curr_reward: -13.0, best_reward: -7.0, running_avg_reward: -15.62, curr_epsilon: 0.0505\n",
      "checkpointing current model weights. highest running_average_reward of -15.55 achieved!\n",
      "episode_num 587, curr_reward: -8.0, best_reward: -7.0, running_avg_reward: -15.55, curr_epsilon: 0.05\n",
      "episode_num 588, curr_reward: -17.0, best_reward: -7.0, running_avg_reward: -15.58, curr_epsilon: 0.0496\n",
      "episode_num 589, curr_reward: -14.0, best_reward: -7.0, running_avg_reward: -15.56, curr_epsilon: 0.0492\n",
      "checkpointing current model weights. highest running_average_reward of -15.49 achieved!\n",
      "episode_num 590, curr_reward: -14.0, best_reward: -7.0, running_avg_reward: -15.49, curr_epsilon: 0.0488\n",
      "checkpointing current model weights. highest running_average_reward of -15.41 achieved!\n",
      "episode_num 591, curr_reward: -8.0, best_reward: -7.0, running_avg_reward: -15.41, curr_epsilon: 0.0483\n",
      "episode_num 592, curr_reward: -17.0, best_reward: -7.0, running_avg_reward: -15.45, curr_epsilon: 0.0479\n",
      "episode_num 593, curr_reward: -12.0, best_reward: -7.0, running_avg_reward: -15.44, curr_epsilon: 0.0475\n",
      "checkpointing current model weights. highest running_average_reward of -15.36 achieved!\n",
      "episode_num 594, curr_reward: -9.0, best_reward: -7.0, running_avg_reward: -15.36, curr_epsilon: 0.0471\n",
      "episode_num 595, curr_reward: -17.0, best_reward: -7.0, running_avg_reward: -15.36, curr_epsilon: 0.0467\n",
      "checkpointing current model weights. highest running_average_reward of -15.29 achieved!\n",
      "episode_num 596, curr_reward: -9.0, best_reward: -7.0, running_avg_reward: -15.29, curr_epsilon: 0.0463\n",
      "checkpointing current model weights. highest running_average_reward of -15.27 achieved!\n",
      "episode_num 597, curr_reward: -13.0, best_reward: -7.0, running_avg_reward: -15.27, curr_epsilon: 0.046\n",
      "checkpointing current model weights. highest running_average_reward of -15.26 achieved!\n",
      "episode_num 598, curr_reward: -13.0, best_reward: -7.0, running_avg_reward: -15.26, curr_epsilon: 0.0457\n",
      "episode_num 599, curr_reward: -15.0, best_reward: -7.0, running_avg_reward: -15.26, curr_epsilon: 0.0454\n",
      "episode_num 600, curr_reward: -15.0, best_reward: -7.0, running_avg_reward: -15.27, curr_epsilon: 0.0451\n",
      "checkpointing current model weights. highest running_average_reward of -15.24 achieved!\n",
      "episode_num 601, curr_reward: -14.0, best_reward: -7.0, running_avg_reward: -15.24, curr_epsilon: 0.0447\n",
      "checkpointing current model weights. highest running_average_reward of -15.19 achieved!\n",
      "episode_num 602, curr_reward: -12.0, best_reward: -7.0, running_avg_reward: -15.19, curr_epsilon: 0.0444\n",
      "checkpointing current model weights. highest running_average_reward of -15.12 achieved!\n",
      "episode_num 603, curr_reward: -11.0, best_reward: -7.0, running_avg_reward: -15.12, curr_epsilon: 0.044\n",
      "checkpointing current model weights. highest running_average_reward of -15.11 achieved!\n",
      "episode_num 604, curr_reward: -14.0, best_reward: -7.0, running_avg_reward: -15.11, curr_epsilon: 0.0437\n",
      "checkpointing current model weights. highest running_average_reward of -15.09 achieved!\n",
      "episode_num 605, curr_reward: -11.0, best_reward: -7.0, running_avg_reward: -15.09, curr_epsilon: 0.0432\n",
      "checkpointing current model weights. highest running_average_reward of -15.06 achieved!\n",
      "episode_num 606, curr_reward: -15.0, best_reward: -7.0, running_avg_reward: -15.06, curr_epsilon: 0.0429\n",
      "checkpointing current model weights. highest running_average_reward of -15.03 achieved!\n",
      "episode_num 607, curr_reward: -13.0, best_reward: -7.0, running_avg_reward: -15.03, curr_epsilon: 0.0426\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpointing current model weights. highest running_average_reward of -14.99 achieved!\n",
      "episode_num 608, curr_reward: -13.0, best_reward: -7.0, running_avg_reward: -14.99, curr_epsilon: 0.0423\n",
      "episode_num 609, curr_reward: -18.0, best_reward: -7.0, running_avg_reward: -15.0, curr_epsilon: 0.042\n",
      "checkpointing current model weights. highest running_average_reward of -14.95 achieved!\n",
      "episode_num 610, curr_reward: -11.0, best_reward: -7.0, running_avg_reward: -14.95, curr_epsilon: 0.0416\n",
      "checkpointing current model weights. highest running_average_reward of -14.87 achieved!\n",
      "episode_num 611, curr_reward: -11.0, best_reward: -7.0, running_avg_reward: -14.87, curr_epsilon: 0.0413\n",
      "checkpointing current model weights. highest running_average_reward of -14.82 achieved!\n",
      "episode_num 612, curr_reward: -14.0, best_reward: -7.0, running_avg_reward: -14.82, curr_epsilon: 0.041\n",
      "checkpointing current model weights. highest running_average_reward of -14.76 achieved!\n",
      "episode_num 613, curr_reward: -12.0, best_reward: -7.0, running_avg_reward: -14.76, curr_epsilon: 0.0406\n",
      "episode_num 614, curr_reward: -16.0, best_reward: -7.0, running_avg_reward: -14.77, curr_epsilon: 0.0403\n",
      "checkpointing current model weights. highest running_average_reward of -14.73 achieved!\n",
      "episode_num 615, curr_reward: -14.0, best_reward: -7.0, running_avg_reward: -14.73, curr_epsilon: 0.0399\n",
      "checkpointing current model weights. highest running_average_reward of -14.69 achieved!\n",
      "episode_num 616, curr_reward: -14.0, best_reward: -7.0, running_avg_reward: -14.69, curr_epsilon: 0.0396\n",
      "episode_num 617, curr_reward: -18.0, best_reward: -7.0, running_avg_reward: -14.72, curr_epsilon: 0.0393\n",
      "episode_num 618, curr_reward: -15.0, best_reward: -7.0, running_avg_reward: -14.73, curr_epsilon: 0.0391\n",
      "checkpointing current model weights. highest running_average_reward of -14.67 achieved!\n",
      "episode_num 619, curr_reward: -11.0, best_reward: -7.0, running_avg_reward: -14.67, curr_epsilon: 0.0388\n",
      "checkpointing current model weights. highest running_average_reward of -14.63 achieved!\n",
      "episode_num 620, curr_reward: -12.0, best_reward: -7.0, running_avg_reward: -14.63, curr_epsilon: 0.0384\n",
      "checkpointing current model weights. highest running_average_reward of -14.55 achieved!\n",
      "episode_num 621, curr_reward: -9.0, best_reward: -7.0, running_avg_reward: -14.55, curr_epsilon: 0.038\n",
      "checkpointing current model weights. highest running_average_reward of -14.54 achieved!\n",
      "episode_num 622, curr_reward: -17.0, best_reward: -7.0, running_avg_reward: -14.54, curr_epsilon: 0.0378\n",
      "checkpointing current model weights. highest running_average_reward of -14.5 achieved!\n",
      "episode_num 623, curr_reward: -12.0, best_reward: -7.0, running_avg_reward: -14.5, curr_epsilon: 0.0374\n",
      "checkpointing current model weights. highest running_average_reward of -14.43 achieved!\n",
      "episode_num 624, curr_reward: -11.0, best_reward: -7.0, running_avg_reward: -14.43, curr_epsilon: 0.0371\n",
      "checkpointing current model weights. highest running_average_reward of -14.38 achieved!\n",
      "episode_num 625, curr_reward: -10.0, best_reward: -7.0, running_avg_reward: -14.38, curr_epsilon: 0.0368\n",
      "checkpointing current model weights. highest running_average_reward of -14.36 achieved!\n",
      "episode_num 626, curr_reward: -13.0, best_reward: -7.0, running_avg_reward: -14.36, curr_epsilon: 0.0365\n",
      "episode_num 627, curr_reward: -18.0, best_reward: -7.0, running_avg_reward: -14.38, curr_epsilon: 0.0363\n",
      "checkpointing current model weights. highest running_average_reward of -14.35 achieved!\n",
      "episode_num 628, curr_reward: -14.0, best_reward: -7.0, running_avg_reward: -14.35, curr_epsilon: 0.036\n",
      "checkpointing current model weights. highest running_average_reward of -14.34 achieved!\n",
      "episode_num 629, curr_reward: -13.0, best_reward: -7.0, running_avg_reward: -14.34, curr_epsilon: 0.0357\n",
      "checkpointing current model weights. highest running_average_reward of -14.28 achieved!\n",
      "episode_num 630, curr_reward: -10.0, best_reward: -7.0, running_avg_reward: -14.28, curr_epsilon: 0.0354\n",
      "episode_num 631, curr_reward: -17.0, best_reward: -7.0, running_avg_reward: -14.29, curr_epsilon: 0.0351\n",
      "checkpointing current model weights. highest running_average_reward of -14.25 achieved!\n",
      "episode_num 632, curr_reward: -12.0, best_reward: -7.0, running_avg_reward: -14.25, curr_epsilon: 0.0349\n",
      "episode_num 633, curr_reward: -17.0, best_reward: -7.0, running_avg_reward: -14.27, curr_epsilon: 0.0346\n",
      "episode_num 634, curr_reward: -15.0, best_reward: -7.0, running_avg_reward: -14.27, curr_epsilon: 0.0344\n",
      "episode_num 635, curr_reward: -14.0, best_reward: -7.0, running_avg_reward: -14.26, curr_epsilon: 0.0341\n",
      "checkpointing current model weights. highest running_average_reward of -14.24 achieved!\n",
      "episode_num 636, curr_reward: -14.0, best_reward: -7.0, running_avg_reward: -14.24, curr_epsilon: 0.0338\n",
      "checkpointing current model weights. highest running_average_reward of -14.18 achieved!\n",
      "episode_num 637, curr_reward: -14.0, best_reward: -7.0, running_avg_reward: -14.18, curr_epsilon: 0.0336\n",
      "checkpointing current model weights. highest running_average_reward of -14.12 achieved!\n",
      "episode_num 638, curr_reward: -10.0, best_reward: -7.0, running_avg_reward: -14.12, curr_epsilon: 0.0333\n",
      "checkpointing current model weights. highest running_average_reward of -14.1 achieved!\n",
      "episode_num 639, curr_reward: -14.0, best_reward: -7.0, running_avg_reward: -14.1, curr_epsilon: 0.0331\n",
      "checkpointing current model weights. highest running_average_reward of -14.03 achieved!\n",
      "episode_num 640, curr_reward: -10.0, best_reward: -7.0, running_avg_reward: -14.03, curr_epsilon: 0.0327\n",
      "checkpointing current model weights. highest running_average_reward of -13.94 achieved!\n",
      "episode_num 641, curr_reward: -10.0, best_reward: -7.0, running_avg_reward: -13.94, curr_epsilon: 0.0325\n",
      "checkpointing current model weights. highest running_average_reward of -13.88 achieved!\n",
      "episode_num 642, curr_reward: -13.0, best_reward: -7.0, running_avg_reward: -13.88, curr_epsilon: 0.0322\n",
      "checkpointing current model weights. highest running_average_reward of -13.79 achieved!\n",
      "episode_num 643, curr_reward: -12.0, best_reward: -7.0, running_avg_reward: -13.79, curr_epsilon: 0.0319\n",
      "checkpointing current model weights. highest running_average_reward of -13.77 achieved!\n",
      "episode_num 644, curr_reward: -15.0, best_reward: -7.0, running_avg_reward: -13.77, curr_epsilon: 0.0317\n",
      "checkpointing current model weights. highest running_average_reward of -13.76 achieved!\n",
      "episode_num 645, curr_reward: -12.0, best_reward: -7.0, running_avg_reward: -13.76, curr_epsilon: 0.0315\n",
      "checkpointing current model weights. highest running_average_reward of -13.74 achieved!\n",
      "episode_num 646, curr_reward: -13.0, best_reward: -7.0, running_avg_reward: -13.74, curr_epsilon: 0.0313\n",
      "episode_num 647, curr_reward: -10.0, best_reward: -7.0, running_avg_reward: -13.76, curr_epsilon: 0.0309\n",
      "checkpointing current model weights. highest running_average_reward of -13.58 achieved!\n",
      "episode_num 648, curr_reward: -2.0, best_reward: -2.0, running_avg_reward: -13.58, curr_epsilon: 0.0306\n",
      "checkpointing current model weights. highest running_average_reward of -13.54 achieved!\n",
      "episode_num 649, curr_reward: -11.0, best_reward: -2.0, running_avg_reward: -13.54, curr_epsilon: 0.0304\n",
      "episode_num 650, curr_reward: -16.0, best_reward: -2.0, running_avg_reward: -13.58, curr_epsilon: 0.0302\n",
      "episode_num 651, curr_reward: -16.0, best_reward: -2.0, running_avg_reward: -13.6, curr_epsilon: 0.03\n",
      "checkpointing current model weights. highest running_average_reward of -13.46 achieved!\n",
      "episode_num 652, curr_reward: -6.0, best_reward: -2.0, running_avg_reward: -13.46, curr_epsilon: 0.0297\n",
      "checkpointing current model weights. highest running_average_reward of -13.38 achieved!\n",
      "episode_num 653, curr_reward: -9.0, best_reward: -2.0, running_avg_reward: -13.38, curr_epsilon: 0.0294\n",
      "episode_num 654, curr_reward: -18.0, best_reward: -2.0, running_avg_reward: -13.43, curr_epsilon: 0.0292\n",
      "episode_num 655, curr_reward: -13.0, best_reward: -2.0, running_avg_reward: -13.39, curr_epsilon: 0.029\n",
      "episode_num 656, curr_reward: -13.0, best_reward: -2.0, running_avg_reward: -13.41, curr_epsilon: 0.0287\n",
      "episode_num 657, curr_reward: -20.0, best_reward: -2.0, running_avg_reward: -13.48, curr_epsilon: 0.0286\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpointing current model weights. highest running_average_reward of -13.31 achieved!\n",
      "episode_num 658, curr_reward: -2.0, best_reward: -2.0, running_avg_reward: -13.31, curr_epsilon: 0.0282\n",
      "checkpointing current model weights. highest running_average_reward of -13.22 achieved!\n",
      "episode_num 659, curr_reward: -6.0, best_reward: -2.0, running_avg_reward: -13.22, curr_epsilon: 0.0279\n",
      "checkpointing current model weights. highest running_average_reward of -13.13 achieved!\n",
      "episode_num 660, curr_reward: -10.0, best_reward: -2.0, running_avg_reward: -13.13, curr_epsilon: 0.0277\n",
      "episode_num 661, curr_reward: -18.0, best_reward: -2.0, running_avg_reward: -13.19, curr_epsilon: 0.0276\n",
      "checkpointing current model weights. highest running_average_reward of -13.1 achieved!\n",
      "episode_num 662, curr_reward: -8.0, best_reward: -2.0, running_avg_reward: -13.1, curr_epsilon: 0.0273\n",
      "checkpointing current model weights. highest running_average_reward of -12.98 achieved!\n",
      "episode_num 663, curr_reward: -7.0, best_reward: -2.0, running_avg_reward: -12.98, curr_epsilon: 0.0271\n",
      "episode_num 664, curr_reward: -18.0, best_reward: -2.0, running_avg_reward: -13.03, curr_epsilon: 0.0269\n",
      "episode_num 665, curr_reward: -12.0, best_reward: -2.0, running_avg_reward: -12.98, curr_epsilon: 0.0268\n",
      "checkpointing current model weights. highest running_average_reward of -12.93 achieved!\n",
      "episode_num 666, curr_reward: -9.0, best_reward: -2.0, running_avg_reward: -12.93, curr_epsilon: 0.0265\n",
      "episode_num 667, curr_reward: -15.0, best_reward: -2.0, running_avg_reward: -12.94, curr_epsilon: 0.0263\n",
      "episode_num 668, curr_reward: -16.0, best_reward: -2.0, running_avg_reward: -12.96, curr_epsilon: 0.0261\n",
      "checkpointing current model weights. highest running_average_reward of -12.88 achieved!\n",
      "episode_num 669, curr_reward: -8.0, best_reward: -2.0, running_avg_reward: -12.88, curr_epsilon: 0.0259\n",
      "checkpointing current model weights. highest running_average_reward of -12.82 achieved!\n",
      "episode_num 670, curr_reward: -7.0, best_reward: -2.0, running_avg_reward: -12.82, curr_epsilon: 0.0257\n",
      "episode_num 671, curr_reward: -16.0, best_reward: -2.0, running_avg_reward: -12.84, curr_epsilon: 0.0255\n",
      "episode_num 672, curr_reward: -11.0, best_reward: -2.0, running_avg_reward: -12.83, curr_epsilon: 0.0253\n",
      "checkpointing current model weights. highest running_average_reward of -12.77 achieved!\n",
      "episode_num 673, curr_reward: -10.0, best_reward: -2.0, running_avg_reward: -12.77, curr_epsilon: 0.0251\n",
      "checkpointing current model weights. highest running_average_reward of -12.76 achieved!\n",
      "episode_num 674, curr_reward: -11.0, best_reward: -2.0, running_avg_reward: -12.76, curr_epsilon: 0.0248\n",
      "checkpointing current model weights. highest running_average_reward of -12.73 achieved!\n",
      "episode_num 675, curr_reward: -13.0, best_reward: -2.0, running_avg_reward: -12.73, curr_epsilon: 0.0246\n",
      "episode_num 676, curr_reward: -15.0, best_reward: -2.0, running_avg_reward: -12.74, curr_epsilon: 0.0244\n",
      "checkpointing current model weights. highest running_average_reward of -12.64 achieved!\n",
      "episode_num 677, curr_reward: -3.0, best_reward: -2.0, running_avg_reward: -12.64, curr_epsilon: 0.0241\n",
      "checkpointing current model weights. highest running_average_reward of -12.51 achieved!\n",
      "episode_num 678, curr_reward: -6.0, best_reward: -2.0, running_avg_reward: -12.51, curr_epsilon: 0.0239\n",
      "checkpointing current model weights. highest running_average_reward of -12.42 achieved!\n",
      "episode_num 679, curr_reward: -8.0, best_reward: -2.0, running_avg_reward: -12.42, curr_epsilon: 0.0237\n",
      "checkpointing current model weights. highest running_average_reward of -12.4 achieved!\n",
      "episode_num 680, curr_reward: -14.0, best_reward: -2.0, running_avg_reward: -12.4, curr_epsilon: 0.0235\n",
      "checkpointing current model weights. highest running_average_reward of -12.29 achieved!\n",
      "episode_num 681, curr_reward: -10.0, best_reward: -2.0, running_avg_reward: -12.29, curr_epsilon: 0.0233\n",
      "episode_num 682, curr_reward: -9.0, best_reward: -2.0, running_avg_reward: -12.31, curr_epsilon: 0.0232\n",
      "episode_num 683, curr_reward: -20.0, best_reward: -2.0, running_avg_reward: -12.37, curr_epsilon: 0.023\n",
      "episode_num 684, curr_reward: -14.0, best_reward: -2.0, running_avg_reward: -12.42, curr_epsilon: 0.0229\n",
      "episode_num 685, curr_reward: -12.0, best_reward: -2.0, running_avg_reward: -12.44, curr_epsilon: 0.0227\n",
      "episode_num 686, curr_reward: -3.0, best_reward: -2.0, running_avg_reward: -12.34, curr_epsilon: 0.0224\n",
      "episode_num 687, curr_reward: -12.0, best_reward: -2.0, running_avg_reward: -12.38, curr_epsilon: 0.0222\n",
      "episode_num 688, curr_reward: -18.0, best_reward: -2.0, running_avg_reward: -12.39, curr_epsilon: 0.0221\n",
      "episode_num 689, curr_reward: -14.0, best_reward: -2.0, running_avg_reward: -12.39, curr_epsilon: 0.022\n",
      "episode_num 690, curr_reward: -10.0, best_reward: -2.0, running_avg_reward: -12.35, curr_epsilon: 0.0218\n",
      "episode_num 691, curr_reward: -11.0, best_reward: -2.0, running_avg_reward: -12.38, curr_epsilon: 0.0216\n",
      "episode_num 692, curr_reward: -9.0, best_reward: -2.0, running_avg_reward: -12.3, curr_epsilon: 0.0215\n",
      "checkpointing current model weights. highest running_average_reward of -12.26 achieved!\n",
      "episode_num 693, curr_reward: -8.0, best_reward: -2.0, running_avg_reward: -12.26, curr_epsilon: 0.0213\n",
      "checkpointing current model weights. highest running_average_reward of -12.23 achieved!\n",
      "episode_num 694, curr_reward: -6.0, best_reward: -2.0, running_avg_reward: -12.23, curr_epsilon: 0.0211\n",
      "checkpointing current model weights. highest running_average_reward of -12.16 achieved!\n",
      "episode_num 695, curr_reward: -10.0, best_reward: -2.0, running_avg_reward: -12.16, curr_epsilon: 0.0209\n",
      "checkpointing current model weights. highest running_average_reward of -12.13 achieved!\n",
      "episode_num 696, curr_reward: -6.0, best_reward: -2.0, running_avg_reward: -12.13, curr_epsilon: 0.0208\n",
      "checkpointing current model weights. highest running_average_reward of -12.06 achieved!\n",
      "episode_num 697, curr_reward: -6.0, best_reward: -2.0, running_avg_reward: -12.06, curr_epsilon: 0.0206\n",
      "episode_num 698, curr_reward: -14.0, best_reward: -2.0, running_avg_reward: -12.07, curr_epsilon: 0.0205\n",
      "checkpointing current model weights. highest running_average_reward of -12.04 achieved!\n",
      "episode_num 699, curr_reward: -12.0, best_reward: -2.0, running_avg_reward: -12.04, curr_epsilon: 0.0203\n",
      "checkpointing current model weights. highest running_average_reward of -11.96 achieved!\n",
      "episode_num 700, curr_reward: -7.0, best_reward: -2.0, running_avg_reward: -11.96, curr_epsilon: 0.0201\n",
      "episode_num 701, curr_reward: -14.0, best_reward: -2.0, running_avg_reward: -11.96, curr_epsilon: 0.02\n",
      "episode_num 702, curr_reward: -16.0, best_reward: -2.0, running_avg_reward: -12.0, curr_epsilon: 0.0199\n",
      "episode_num 703, curr_reward: -9.0, best_reward: -2.0, running_avg_reward: -11.98, curr_epsilon: 0.0197\n",
      "episode_num 704, curr_reward: -14.0, best_reward: -2.0, running_avg_reward: -11.98, curr_epsilon: 0.0196\n",
      "episode_num 705, curr_reward: -16.0, best_reward: -2.0, running_avg_reward: -12.03, curr_epsilon: 0.0195\n",
      "episode_num 706, curr_reward: -15.0, best_reward: -2.0, running_avg_reward: -12.03, curr_epsilon: 0.0194\n",
      "episode_num 707, curr_reward: -12.0, best_reward: -2.0, running_avg_reward: -12.02, curr_epsilon: 0.0192\n",
      "checkpointing current model weights. highest running_average_reward of -11.92 achieved!\n",
      "episode_num 708, curr_reward: -3.0, best_reward: -2.0, running_avg_reward: -11.92, curr_epsilon: 0.0191\n",
      "checkpointing current model weights. highest running_average_reward of -11.88 achieved!\n",
      "episode_num 709, curr_reward: -14.0, best_reward: -2.0, running_avg_reward: -11.88, curr_epsilon: 0.019\n",
      "episode_num 710, curr_reward: -17.0, best_reward: -2.0, running_avg_reward: -11.94, curr_epsilon: 0.0189\n",
      "episode_num 711, curr_reward: -11.0, best_reward: -2.0, running_avg_reward: -11.94, curr_epsilon: 0.0187\n",
      "episode_num 712, curr_reward: -12.0, best_reward: -2.0, running_avg_reward: -11.92, curr_epsilon: 0.0186\n",
      "episode_num 713, curr_reward: -13.0, best_reward: -2.0, running_avg_reward: -11.93, curr_epsilon: 0.0185\n",
      "episode_num 714, curr_reward: -12.0, best_reward: -2.0, running_avg_reward: -11.89, curr_epsilon: 0.0183\n",
      "episode_num 715, curr_reward: -14.0, best_reward: -2.0, running_avg_reward: -11.89, curr_epsilon: 0.0182\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpointing current model weights. highest running_average_reward of -11.83 achieved!\n",
      "episode_num 716, curr_reward: -8.0, best_reward: -2.0, running_avg_reward: -11.83, curr_epsilon: 0.0181\n",
      "checkpointing current model weights. highest running_average_reward of -11.76 achieved!\n",
      "episode_num 717, curr_reward: -11.0, best_reward: -2.0, running_avg_reward: -11.76, curr_epsilon: 0.0179\n",
      "episode_num 718, curr_reward: -18.0, best_reward: -2.0, running_avg_reward: -11.79, curr_epsilon: 0.0178\n",
      "checkpointing current model weights. highest running_average_reward of -11.7 achieved!\n",
      "episode_num 719, curr_reward: -2.0, best_reward: -2.0, running_avg_reward: -11.7, curr_epsilon: 0.0176\n",
      "episode_num 720, curr_reward: -14.0, best_reward: -2.0, running_avg_reward: -11.72, curr_epsilon: 0.0175\n",
      "episode_num 721, curr_reward: -7.0, best_reward: -2.0, running_avg_reward: -11.7, curr_epsilon: 0.0174\n",
      "checkpointing current model weights. highest running_average_reward of -11.66 achieved!\n",
      "episode_num 722, curr_reward: -13.0, best_reward: -2.0, running_avg_reward: -11.66, curr_epsilon: 0.0172\n",
      "episode_num 723, curr_reward: -12.0, best_reward: -2.0, running_avg_reward: -11.66, curr_epsilon: 0.0171\n",
      "episode_num 724, curr_reward: -14.0, best_reward: -2.0, running_avg_reward: -11.69, curr_epsilon: 0.017\n",
      "episode_num 725, curr_reward: -18.0, best_reward: -2.0, running_avg_reward: -11.77, curr_epsilon: 0.0169\n",
      "episode_num 726, curr_reward: -14.0, best_reward: -2.0, running_avg_reward: -11.78, curr_epsilon: 0.0168\n",
      "episode_num 727, curr_reward: -12.0, best_reward: -2.0, running_avg_reward: -11.72, curr_epsilon: 0.0167\n",
      "episode_num 728, curr_reward: -14.0, best_reward: -2.0, running_avg_reward: -11.72, curr_epsilon: 0.0166\n",
      "episode_num 729, curr_reward: -9.0, best_reward: -2.0, running_avg_reward: -11.68, curr_epsilon: 0.0165\n",
      "episode_num 730, curr_reward: -18.0, best_reward: -2.0, running_avg_reward: -11.76, curr_epsilon: 0.0164\n",
      "episode_num 731, curr_reward: -14.0, best_reward: -2.0, running_avg_reward: -11.73, curr_epsilon: 0.0163\n",
      "episode_num 732, curr_reward: -14.0, best_reward: -2.0, running_avg_reward: -11.75, curr_epsilon: 0.0162\n",
      "episode_num 733, curr_reward: -8.0, best_reward: -2.0, running_avg_reward: -11.66, curr_epsilon: 0.0161\n",
      "checkpointing current model weights. highest running_average_reward of -11.55 achieved!\n",
      "episode_num 734, curr_reward: -4.0, best_reward: -2.0, running_avg_reward: -11.55, curr_epsilon: 0.016\n",
      "episode_num 735, curr_reward: -14.0, best_reward: -2.0, running_avg_reward: -11.55, curr_epsilon: 0.0159\n",
      "checkpointing current model weights. highest running_average_reward of -11.52 achieved!\n",
      "episode_num 736, curr_reward: -11.0, best_reward: -2.0, running_avg_reward: -11.52, curr_epsilon: 0.0157\n",
      "checkpointing current model weights. highest running_average_reward of -11.49 achieved!\n",
      "episode_num 737, curr_reward: -11.0, best_reward: -2.0, running_avg_reward: -11.49, curr_epsilon: 0.0157\n",
      "checkpointing current model weights. highest running_average_reward of -11.48 achieved!\n",
      "episode_num 738, curr_reward: -9.0, best_reward: -2.0, running_avg_reward: -11.48, curr_epsilon: 0.0155\n",
      "episode_num 739, curr_reward: -19.0, best_reward: -2.0, running_avg_reward: -11.53, curr_epsilon: 0.0155\n",
      "episode_num 740, curr_reward: -6.0, best_reward: -2.0, running_avg_reward: -11.49, curr_epsilon: 0.0154\n",
      "checkpointing current model weights. highest running_average_reward of -11.47 achieved!\n",
      "episode_num 741, curr_reward: -8.0, best_reward: -2.0, running_avg_reward: -11.47, curr_epsilon: 0.0153\n",
      "episode_num 742, curr_reward: -18.0, best_reward: -2.0, running_avg_reward: -11.52, curr_epsilon: 0.0152\n",
      "episode_num 743, curr_reward: -18.0, best_reward: -2.0, running_avg_reward: -11.58, curr_epsilon: 0.0151\n",
      "episode_num 744, curr_reward: -12.0, best_reward: -2.0, running_avg_reward: -11.55, curr_epsilon: 0.015\n",
      "checkpointing current model weights. highest running_average_reward of -11.44 achieved!\n",
      "episode_num 745, curr_reward: -1.0, best_reward: -1.0, running_avg_reward: -11.44, curr_epsilon: 0.0149\n",
      "episode_num 746, curr_reward: -14.0, best_reward: -1.0, running_avg_reward: -11.45, curr_epsilon: 0.0148\n",
      "episode_num 747, curr_reward: -12.0, best_reward: -1.0, running_avg_reward: -11.47, curr_epsilon: 0.0147\n",
      "episode_num 748, curr_reward: -12.0, best_reward: -1.0, running_avg_reward: -11.57, curr_epsilon: 0.0146\n",
      "episode_num 749, curr_reward: -6.0, best_reward: -1.0, running_avg_reward: -11.52, curr_epsilon: 0.0145\n",
      "episode_num 750, curr_reward: -11.0, best_reward: -1.0, running_avg_reward: -11.47, curr_epsilon: 0.0144\n",
      "checkpointing current model weights. highest running_average_reward of -11.4 achieved!\n",
      "episode_num 751, curr_reward: -9.0, best_reward: -1.0, running_avg_reward: -11.4, curr_epsilon: 0.0143\n",
      "episode_num 752, curr_reward: -12.0, best_reward: -1.0, running_avg_reward: -11.46, curr_epsilon: 0.0142\n",
      "episode_num 753, curr_reward: -9.0, best_reward: -1.0, running_avg_reward: -11.46, curr_epsilon: 0.0141\n",
      "checkpointing current model weights. highest running_average_reward of -11.37 achieved!\n",
      "episode_num 754, curr_reward: -9.0, best_reward: -1.0, running_avg_reward: -11.37, curr_epsilon: 0.014\n",
      "episode_num 755, curr_reward: -13.0, best_reward: -1.0, running_avg_reward: -11.37, curr_epsilon: 0.0139\n",
      "checkpointing current model weights. highest running_average_reward of -11.33 achieved!\n",
      "episode_num 756, curr_reward: -9.0, best_reward: -1.0, running_avg_reward: -11.33, curr_epsilon: 0.0139\n",
      "checkpointing current model weights. highest running_average_reward of -11.18 achieved!\n",
      "episode_num 757, curr_reward: -5.0, best_reward: -1.0, running_avg_reward: -11.18, curr_epsilon: 0.0137\n",
      "episode_num 758, curr_reward: -15.0, best_reward: -1.0, running_avg_reward: -11.31, curr_epsilon: 0.0137\n",
      "episode_num 759, curr_reward: -8.0, best_reward: -1.0, running_avg_reward: -11.33, curr_epsilon: 0.0136\n",
      "episode_num 760, curr_reward: -16.0, best_reward: -1.0, running_avg_reward: -11.39, curr_epsilon: 0.0135\n",
      "episode_num 761, curr_reward: -15.0, best_reward: -1.0, running_avg_reward: -11.36, curr_epsilon: 0.0134\n",
      "episode_num 762, curr_reward: -12.0, best_reward: -1.0, running_avg_reward: -11.4, curr_epsilon: 0.0134\n",
      "episode_num 763, curr_reward: -17.0, best_reward: -1.0, running_avg_reward: -11.5, curr_epsilon: 0.0133\n",
      "episode_num 764, curr_reward: -12.0, best_reward: -1.0, running_avg_reward: -11.44, curr_epsilon: 0.0132\n",
      "episode_num 765, curr_reward: -9.0, best_reward: -1.0, running_avg_reward: -11.41, curr_epsilon: 0.0131\n",
      "episode_num 766, curr_reward: -14.0, best_reward: -1.0, running_avg_reward: -11.46, curr_epsilon: 0.0131\n",
      "episode_num 767, curr_reward: -4.0, best_reward: -1.0, running_avg_reward: -11.35, curr_epsilon: 0.013\n",
      "episode_num 768, curr_reward: -16.0, best_reward: -1.0, running_avg_reward: -11.35, curr_epsilon: 0.0129\n",
      "episode_num 769, curr_reward: -14.0, best_reward: -1.0, running_avg_reward: -11.41, curr_epsilon: 0.0128\n",
      "episode_num 770, curr_reward: -12.0, best_reward: -1.0, running_avg_reward: -11.46, curr_epsilon: 0.0127\n",
      "episode_num 771, curr_reward: -6.0, best_reward: -1.0, running_avg_reward: -11.36, curr_epsilon: 0.0126\n",
      "episode_num 772, curr_reward: -12.0, best_reward: -1.0, running_avg_reward: -11.37, curr_epsilon: 0.0126\n",
      "episode_num 773, curr_reward: -12.0, best_reward: -1.0, running_avg_reward: -11.39, curr_epsilon: 0.0125\n",
      "episode_num 774, curr_reward: -12.0, best_reward: -1.0, running_avg_reward: -11.4, curr_epsilon: 0.0124\n",
      "episode_num 775, curr_reward: 5.0, best_reward: 5.0, running_avg_reward: -11.22, curr_epsilon: 0.0123\n",
      "episode_num 776, curr_reward: -16.0, best_reward: 5.0, running_avg_reward: -11.23, curr_epsilon: 0.0122\n",
      "episode_num 777, curr_reward: -9.0, best_reward: 5.0, running_avg_reward: -11.29, curr_epsilon: 0.0122\n",
      "episode_num 778, curr_reward: -6.0, best_reward: 5.0, running_avg_reward: -11.29, curr_epsilon: 0.0121\n",
      "checkpointing current model weights. highest running_average_reward of -11.15 achieved!\n",
      "episode_num 779, curr_reward: 6.0, best_reward: 6.0, running_avg_reward: -11.15, curr_epsilon: 0.012\n",
      "checkpointing current model weights. highest running_average_reward of -11.13 achieved!\n",
      "episode_num 780, curr_reward: -12.0, best_reward: 6.0, running_avg_reward: -11.13, curr_epsilon: 0.0119\n",
      "episode_num 781, curr_reward: -18.0, best_reward: 6.0, running_avg_reward: -11.21, curr_epsilon: 0.0119\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 782, curr_reward: -6.0, best_reward: 6.0, running_avg_reward: -11.18, curr_epsilon: 0.0118\n",
      "checkpointing current model weights. highest running_average_reward of -11.03 achieved!\n",
      "episode_num 783, curr_reward: -5.0, best_reward: 6.0, running_avg_reward: -11.03, curr_epsilon: 0.0117\n",
      "checkpointing current model weights. highest running_average_reward of -10.97 achieved!\n",
      "episode_num 784, curr_reward: -8.0, best_reward: 6.0, running_avg_reward: -10.97, curr_epsilon: 0.0116\n",
      "checkpointing current model weights. highest running_average_reward of -10.93 achieved!\n",
      "episode_num 785, curr_reward: -8.0, best_reward: 6.0, running_avg_reward: -10.93, curr_epsilon: 0.0115\n",
      "episode_num 786, curr_reward: -18.0, best_reward: 6.0, running_avg_reward: -11.08, curr_epsilon: 0.0115\n",
      "episode_num 787, curr_reward: -8.0, best_reward: 6.0, running_avg_reward: -11.04, curr_epsilon: 0.0114\n",
      "checkpointing current model weights. highest running_average_reward of -10.92 achieved!\n",
      "episode_num 788, curr_reward: -6.0, best_reward: 6.0, running_avg_reward: -10.92, curr_epsilon: 0.0113\n",
      "episode_num 789, curr_reward: -15.0, best_reward: 6.0, running_avg_reward: -10.93, curr_epsilon: 0.0113\n",
      "episode_num 790, curr_reward: -16.0, best_reward: 6.0, running_avg_reward: -10.99, curr_epsilon: 0.0112\n",
      "episode_num 791, curr_reward: -7.0, best_reward: 6.0, running_avg_reward: -10.95, curr_epsilon: 0.0111\n",
      "episode_num 792, curr_reward: -17.0, best_reward: 6.0, running_avg_reward: -11.03, curr_epsilon: 0.0111\n",
      "episode_num 793, curr_reward: -7.0, best_reward: 6.0, running_avg_reward: -11.02, curr_epsilon: 0.011\n",
      "episode_num 794, curr_reward: -6.0, best_reward: 6.0, running_avg_reward: -11.02, curr_epsilon: 0.0109\n",
      "episode_num 795, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -11.01, curr_epsilon: 0.0109\n",
      "episode_num 796, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -11.04, curr_epsilon: 0.0108\n",
      "episode_num 797, curr_reward: -15.0, best_reward: 6.0, running_avg_reward: -11.13, curr_epsilon: 0.0107\n",
      "episode_num 798, curr_reward: -15.0, best_reward: 6.0, running_avg_reward: -11.14, curr_epsilon: 0.0107\n",
      "episode_num 799, curr_reward: -10.0, best_reward: 6.0, running_avg_reward: -11.12, curr_epsilon: 0.0106\n",
      "episode_num 800, curr_reward: -11.0, best_reward: 6.0, running_avg_reward: -11.16, curr_epsilon: 0.0106\n",
      "episode_num 801, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -11.16, curr_epsilon: 0.0105\n",
      "episode_num 802, curr_reward: -7.0, best_reward: 6.0, running_avg_reward: -11.07, curr_epsilon: 0.0104\n",
      "episode_num 803, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -11.12, curr_epsilon: 0.0104\n",
      "episode_num 804, curr_reward: -12.0, best_reward: 6.0, running_avg_reward: -11.1, curr_epsilon: 0.0104\n",
      "episode_num 805, curr_reward: -12.0, best_reward: 6.0, running_avg_reward: -11.06, curr_epsilon: 0.0103\n",
      "episode_num 806, curr_reward: -15.0, best_reward: 6.0, running_avg_reward: -11.06, curr_epsilon: 0.0103\n",
      "episode_num 807, curr_reward: -10.0, best_reward: 6.0, running_avg_reward: -11.04, curr_epsilon: 0.0102\n",
      "episode_num 808, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -11.15, curr_epsilon: 0.0102\n",
      "episode_num 809, curr_reward: -15.0, best_reward: 6.0, running_avg_reward: -11.16, curr_epsilon: 0.0101\n",
      "episode_num 810, curr_reward: -15.0, best_reward: 6.0, running_avg_reward: -11.14, curr_epsilon: 0.0101\n",
      "episode_num 811, curr_reward: -5.0, best_reward: 6.0, running_avg_reward: -11.08, curr_epsilon: 0.01\n",
      "episode_num 812, curr_reward: -11.0, best_reward: 6.0, running_avg_reward: -11.07, curr_epsilon: 0.0099\n",
      "episode_num 813, curr_reward: -6.0, best_reward: 6.0, running_avg_reward: -11.0, curr_epsilon: 0.0099\n",
      "episode_num 814, curr_reward: -10.0, best_reward: 6.0, running_avg_reward: -10.98, curr_epsilon: 0.0098\n",
      "checkpointing current model weights. highest running_average_reward of -10.89 achieved!\n",
      "episode_num 815, curr_reward: -5.0, best_reward: 6.0, running_avg_reward: -10.89, curr_epsilon: 0.0098\n",
      "episode_num 816, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -10.9, curr_epsilon: 0.0097\n",
      "episode_num 817, curr_reward: -13.0, best_reward: 6.0, running_avg_reward: -10.92, curr_epsilon: 0.0097\n",
      "checkpointing current model weights. highest running_average_reward of -10.87 achieved!\n",
      "episode_num 818, curr_reward: -13.0, best_reward: 6.0, running_avg_reward: -10.87, curr_epsilon: 0.0096\n",
      "episode_num 819, curr_reward: -12.0, best_reward: 6.0, running_avg_reward: -10.97, curr_epsilon: 0.0096\n",
      "episode_num 820, curr_reward: -13.0, best_reward: 6.0, running_avg_reward: -10.96, curr_epsilon: 0.0095\n",
      "episode_num 821, curr_reward: -4.0, best_reward: 6.0, running_avg_reward: -10.93, curr_epsilon: 0.0095\n",
      "episode_num 822, curr_reward: -12.0, best_reward: 6.0, running_avg_reward: -10.92, curr_epsilon: 0.0094\n",
      "checkpointing current model weights. highest running_average_reward of -10.84 achieved!\n",
      "episode_num 823, curr_reward: -4.0, best_reward: 6.0, running_avg_reward: -10.84, curr_epsilon: 0.0094\n",
      "episode_num 824, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -10.84, curr_epsilon: 0.0093\n",
      "checkpointing current model weights. highest running_average_reward of -10.75 achieved!\n",
      "episode_num 825, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -10.75, curr_epsilon: 0.0093\n",
      "checkpointing current model weights. highest running_average_reward of -10.68 achieved!\n",
      "episode_num 826, curr_reward: -7.0, best_reward: 6.0, running_avg_reward: -10.68, curr_epsilon: 0.0092\n",
      "episode_num 827, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -10.7, curr_epsilon: 0.0092\n",
      "checkpointing current model weights. highest running_average_reward of -10.6 achieved!\n",
      "episode_num 828, curr_reward: -4.0, best_reward: 6.0, running_avg_reward: -10.6, curr_epsilon: 0.0091\n",
      "episode_num 829, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -10.65, curr_epsilon: 0.0091\n",
      "episode_num 830, curr_reward: -15.0, best_reward: 6.0, running_avg_reward: -10.62, curr_epsilon: 0.0091\n",
      "checkpointing current model weights. highest running_average_reward of -10.57 achieved!\n",
      "episode_num 831, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -10.57, curr_epsilon: 0.009\n",
      "checkpointing current model weights. highest running_average_reward of -10.46 achieved!\n",
      "episode_num 832, curr_reward: -3.0, best_reward: 6.0, running_avg_reward: -10.46, curr_epsilon: 0.009\n",
      "checkpointing current model weights. highest running_average_reward of -10.41 achieved!\n",
      "episode_num 833, curr_reward: -3.0, best_reward: 6.0, running_avg_reward: -10.41, curr_epsilon: 0.0089\n",
      "episode_num 834, curr_reward: -11.0, best_reward: 6.0, running_avg_reward: -10.48, curr_epsilon: 0.0089\n",
      "episode_num 835, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -10.48, curr_epsilon: 0.0088\n",
      "episode_num 836, curr_reward: -8.0, best_reward: 6.0, running_avg_reward: -10.45, curr_epsilon: 0.0088\n",
      "episode_num 837, curr_reward: -8.0, best_reward: 6.0, running_avg_reward: -10.42, curr_epsilon: 0.0088\n",
      "episode_num 838, curr_reward: -15.0, best_reward: 6.0, running_avg_reward: -10.48, curr_epsilon: 0.0087\n",
      "checkpointing current model weights. highest running_average_reward of -10.27 achieved!\n",
      "episode_num 839, curr_reward: 2.0, best_reward: 6.0, running_avg_reward: -10.27, curr_epsilon: 0.0087\n",
      "episode_num 840, curr_reward: -12.0, best_reward: 6.0, running_avg_reward: -10.33, curr_epsilon: 0.0086\n",
      "episode_num 841, curr_reward: -8.0, best_reward: 6.0, running_avg_reward: -10.33, curr_epsilon: 0.0086\n",
      "checkpointing current model weights. highest running_average_reward of -10.2 achieved!\n",
      "episode_num 842, curr_reward: -5.0, best_reward: 6.0, running_avg_reward: -10.2, curr_epsilon: 0.0085\n",
      "checkpointing current model weights. highest running_average_reward of -10.16 achieved!\n",
      "episode_num 843, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -10.16, curr_epsilon: 0.0085\n",
      "episode_num 844, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -10.18, curr_epsilon: 0.0085\n",
      "episode_num 845, curr_reward: -12.0, best_reward: 6.0, running_avg_reward: -10.29, curr_epsilon: 0.0085\n",
      "episode_num 846, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -10.24, curr_epsilon: 0.0084\n",
      "episode_num 847, curr_reward: -13.0, best_reward: 6.0, running_avg_reward: -10.25, curr_epsilon: 0.0084\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpointing current model weights. highest running_average_reward of -10.14 achieved!\n",
      "episode_num 848, curr_reward: -1.0, best_reward: 6.0, running_avg_reward: -10.14, curr_epsilon: 0.0084\n",
      "episode_num 849, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -10.17, curr_epsilon: 0.0083\n",
      "episode_num 850, curr_reward: -8.0, best_reward: 6.0, running_avg_reward: -10.14, curr_epsilon: 0.0083\n",
      "episode_num 851, curr_reward: -11.0, best_reward: 6.0, running_avg_reward: -10.16, curr_epsilon: 0.0082\n",
      "checkpointing current model weights. highest running_average_reward of -10.07 achieved!\n",
      "episode_num 852, curr_reward: -3.0, best_reward: 6.0, running_avg_reward: -10.07, curr_epsilon: 0.0082\n",
      "episode_num 853, curr_reward: -11.0, best_reward: 6.0, running_avg_reward: -10.09, curr_epsilon: 0.0082\n",
      "episode_num 854, curr_reward: -11.0, best_reward: 6.0, running_avg_reward: -10.11, curr_epsilon: 0.0082\n",
      "episode_num 855, curr_reward: -13.0, best_reward: 6.0, running_avg_reward: -10.11, curr_epsilon: 0.0081\n",
      "episode_num 856, curr_reward: -11.0, best_reward: 6.0, running_avg_reward: -10.13, curr_epsilon: 0.0081\n",
      "episode_num 857, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -10.22, curr_epsilon: 0.0081\n",
      "episode_num 858, curr_reward: -7.0, best_reward: 6.0, running_avg_reward: -10.14, curr_epsilon: 0.008\n",
      "episode_num 859, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -10.2, curr_epsilon: 0.008\n",
      "episode_num 860, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -10.13, curr_epsilon: 0.008\n",
      "checkpointing current model weights. highest running_average_reward of -10.02 achieved!\n",
      "episode_num 861, curr_reward: -4.0, best_reward: 6.0, running_avg_reward: -10.02, curr_epsilon: 0.0079\n",
      "episode_num 862, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -10.04, curr_epsilon: 0.0079\n",
      "checkpointing current model weights. highest running_average_reward of -10.0 achieved!\n",
      "episode_num 863, curr_reward: -13.0, best_reward: 6.0, running_avg_reward: -10.0, curr_epsilon: 0.0079\n",
      "episode_num 864, curr_reward: -12.0, best_reward: 6.0, running_avg_reward: -10.0, curr_epsilon: 0.0079\n",
      "checkpointing current model weights. highest running_average_reward of -9.95 achieved!\n",
      "episode_num 865, curr_reward: -4.0, best_reward: 6.0, running_avg_reward: -9.95, curr_epsilon: 0.0078\n",
      "checkpointing current model weights. highest running_average_reward of -9.9 achieved!\n",
      "episode_num 866, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -9.9, curr_epsilon: 0.0078\n",
      "checkpointing current model weights. highest running_average_reward of -9.85 achieved!\n",
      "episode_num 867, curr_reward: 1.0, best_reward: 6.0, running_avg_reward: -9.85, curr_epsilon: 0.0078\n",
      "checkpointing current model weights. highest running_average_reward of -9.73 achieved!\n",
      "episode_num 868, curr_reward: -4.0, best_reward: 6.0, running_avg_reward: -9.73, curr_epsilon: 0.0077\n",
      "checkpointing current model weights. highest running_average_reward of -9.71 achieved!\n",
      "episode_num 869, curr_reward: -12.0, best_reward: 6.0, running_avg_reward: -9.71, curr_epsilon: 0.0077\n",
      "checkpointing current model weights. highest running_average_reward of -9.68 achieved!\n",
      "episode_num 870, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -9.68, curr_epsilon: 0.0077\n",
      "checkpointing current model weights. highest running_average_reward of -9.65 achieved!\n",
      "episode_num 871, curr_reward: -3.0, best_reward: 6.0, running_avg_reward: -9.65, curr_epsilon: 0.0076\n",
      "checkpointing current model weights. highest running_average_reward of -9.64 achieved!\n",
      "episode_num 872, curr_reward: -11.0, best_reward: 6.0, running_avg_reward: -9.64, curr_epsilon: 0.0076\n",
      "checkpointing current model weights. highest running_average_reward of -9.49 achieved!\n",
      "episode_num 873, curr_reward: 3.0, best_reward: 6.0, running_avg_reward: -9.49, curr_epsilon: 0.0076\n",
      "checkpointing current model weights. highest running_average_reward of -9.45 achieved!\n",
      "episode_num 874, curr_reward: -8.0, best_reward: 6.0, running_avg_reward: -9.45, curr_epsilon: 0.0075\n",
      "episode_num 875, curr_reward: -5.0, best_reward: 6.0, running_avg_reward: -9.55, curr_epsilon: 0.0075\n",
      "episode_num 876, curr_reward: -20.0, best_reward: 6.0, running_avg_reward: -9.59, curr_epsilon: 0.0075\n",
      "episode_num 877, curr_reward: -3.0, best_reward: 6.0, running_avg_reward: -9.53, curr_epsilon: 0.0075\n",
      "episode_num 878, curr_reward: -13.0, best_reward: 6.0, running_avg_reward: -9.6, curr_epsilon: 0.0075\n",
      "episode_num 879, curr_reward: -10.0, best_reward: 6.0, running_avg_reward: -9.76, curr_epsilon: 0.0074\n",
      "episode_num 880, curr_reward: -4.0, best_reward: 6.0, running_avg_reward: -9.68, curr_epsilon: 0.0074\n",
      "episode_num 881, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -9.59, curr_epsilon: 0.0074\n",
      "episode_num 882, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -9.62, curr_epsilon: 0.0073\n",
      "episode_num 883, curr_reward: -10.0, best_reward: 6.0, running_avg_reward: -9.67, curr_epsilon: 0.0073\n",
      "episode_num 884, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -9.68, curr_epsilon: 0.0073\n",
      "episode_num 885, curr_reward: -15.0, best_reward: 6.0, running_avg_reward: -9.75, curr_epsilon: 0.0073\n",
      "episode_num 886, curr_reward: -10.0, best_reward: 6.0, running_avg_reward: -9.67, curr_epsilon: 0.0073\n",
      "episode_num 887, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -9.68, curr_epsilon: 0.0072\n",
      "episode_num 888, curr_reward: -13.0, best_reward: 6.0, running_avg_reward: -9.75, curr_epsilon: 0.0072\n",
      "episode_num 889, curr_reward: -3.0, best_reward: 6.0, running_avg_reward: -9.63, curr_epsilon: 0.0072\n",
      "episode_num 890, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -9.61, curr_epsilon: 0.0072\n",
      "episode_num 891, curr_reward: -6.0, best_reward: 6.0, running_avg_reward: -9.6, curr_epsilon: 0.0071\n",
      "episode_num 892, curr_reward: -6.0, best_reward: 6.0, running_avg_reward: -9.49, curr_epsilon: 0.0071\n",
      "episode_num 893, curr_reward: -11.0, best_reward: 6.0, running_avg_reward: -9.53, curr_epsilon: 0.0071\n",
      "episode_num 894, curr_reward: -13.0, best_reward: 6.0, running_avg_reward: -9.6, curr_epsilon: 0.0071\n",
      "episode_num 895, curr_reward: -4.0, best_reward: 6.0, running_avg_reward: -9.55, curr_epsilon: 0.0071\n",
      "episode_num 896, curr_reward: -13.0, best_reward: 6.0, running_avg_reward: -9.59, curr_epsilon: 0.007\n",
      "episode_num 897, curr_reward: -13.0, best_reward: 6.0, running_avg_reward: -9.57, curr_epsilon: 0.007\n",
      "episode_num 898, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -9.51, curr_epsilon: 0.007\n",
      "episode_num 899, curr_reward: -11.0, best_reward: 6.0, running_avg_reward: -9.52, curr_epsilon: 0.007\n",
      "episode_num 900, curr_reward: -10.0, best_reward: 6.0, running_avg_reward: -9.51, curr_epsilon: 0.007\n",
      "episode_num 901, curr_reward: -13.0, best_reward: 6.0, running_avg_reward: -9.5, curr_epsilon: 0.0069\n",
      "episode_num 902, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -9.57, curr_epsilon: 0.0069\n",
      "episode_num 903, curr_reward: -13.0, best_reward: 6.0, running_avg_reward: -9.56, curr_epsilon: 0.0069\n",
      "episode_num 904, curr_reward: -10.0, best_reward: 6.0, running_avg_reward: -9.54, curr_epsilon: 0.0069\n",
      "episode_num 905, curr_reward: -16.0, best_reward: 6.0, running_avg_reward: -9.58, curr_epsilon: 0.0069\n",
      "episode_num 906, curr_reward: -7.0, best_reward: 6.0, running_avg_reward: -9.5, curr_epsilon: 0.0069\n",
      "episode_num 907, curr_reward: -13.0, best_reward: 6.0, running_avg_reward: -9.53, curr_epsilon: 0.0068\n",
      "episode_num 908, curr_reward: -15.0, best_reward: 6.0, running_avg_reward: -9.54, curr_epsilon: 0.0068\n",
      "checkpointing current model weights. highest running_average_reward of -9.43 achieved!\n",
      "episode_num 909, curr_reward: -4.0, best_reward: 6.0, running_avg_reward: -9.43, curr_epsilon: 0.0068\n",
      "checkpointing current model weights. highest running_average_reward of -9.38 achieved!\n",
      "episode_num 910, curr_reward: -10.0, best_reward: 6.0, running_avg_reward: -9.38, curr_epsilon: 0.0068\n",
      "episode_num 911, curr_reward: -11.0, best_reward: 6.0, running_avg_reward: -9.44, curr_epsilon: 0.0068\n",
      "checkpointing current model weights. highest running_average_reward of -9.37 achieved!\n",
      "episode_num 912, curr_reward: -4.0, best_reward: 6.0, running_avg_reward: -9.37, curr_epsilon: 0.0067\n",
      "episode_num 913, curr_reward: -8.0, best_reward: 6.0, running_avg_reward: -9.39, curr_epsilon: 0.0067\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 914, curr_reward: -17.0, best_reward: 6.0, running_avg_reward: -9.46, curr_epsilon: 0.0067\n",
      "episode_num 915, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -9.55, curr_epsilon: 0.0067\n",
      "episode_num 916, curr_reward: 5.0, best_reward: 6.0, running_avg_reward: -9.41, curr_epsilon: 0.0067\n",
      "episode_num 917, curr_reward: -10.0, best_reward: 6.0, running_avg_reward: -9.38, curr_epsilon: 0.0067\n",
      "checkpointing current model weights. highest running_average_reward of -9.34 achieved!\n",
      "episode_num 918, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -9.34, curr_epsilon: 0.0066\n",
      "episode_num 919, curr_reward: -15.0, best_reward: 6.0, running_avg_reward: -9.37, curr_epsilon: 0.0066\n",
      "checkpointing current model weights. highest running_average_reward of -9.33 achieved!\n",
      "episode_num 920, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -9.33, curr_epsilon: 0.0066\n",
      "episode_num 921, curr_reward: -6.0, best_reward: 6.0, running_avg_reward: -9.35, curr_epsilon: 0.0066\n",
      "episode_num 922, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -9.37, curr_epsilon: 0.0066\n",
      "episode_num 923, curr_reward: -17.0, best_reward: 6.0, running_avg_reward: -9.5, curr_epsilon: 0.0066\n",
      "episode_num 924, curr_reward: -8.0, best_reward: 6.0, running_avg_reward: -9.44, curr_epsilon: 0.0065\n",
      "episode_num 925, curr_reward: -15.0, best_reward: 6.0, running_avg_reward: -9.5, curr_epsilon: 0.0065\n",
      "episode_num 926, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -9.57, curr_epsilon: 0.0065\n",
      "episode_num 927, curr_reward: -12.0, best_reward: 6.0, running_avg_reward: -9.55, curr_epsilon: 0.0065\n",
      "episode_num 928, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -9.65, curr_epsilon: 0.0065\n",
      "episode_num 929, curr_reward: -7.0, best_reward: 6.0, running_avg_reward: -9.58, curr_epsilon: 0.0065\n",
      "episode_num 930, curr_reward: -5.0, best_reward: 6.0, running_avg_reward: -9.48, curr_epsilon: 0.0065\n",
      "episode_num 931, curr_reward: -11.0, best_reward: 6.0, running_avg_reward: -9.5, curr_epsilon: 0.0064\n",
      "episode_num 932, curr_reward: -11.0, best_reward: 6.0, running_avg_reward: -9.58, curr_epsilon: 0.0064\n",
      "episode_num 933, curr_reward: -10.0, best_reward: 6.0, running_avg_reward: -9.65, curr_epsilon: 0.0064\n",
      "episode_num 934, curr_reward: -5.0, best_reward: 6.0, running_avg_reward: -9.59, curr_epsilon: 0.0064\n",
      "episode_num 935, curr_reward: -16.0, best_reward: 6.0, running_avg_reward: -9.61, curr_epsilon: 0.0064\n",
      "episode_num 936, curr_reward: -5.0, best_reward: 6.0, running_avg_reward: -9.58, curr_epsilon: 0.0064\n",
      "episode_num 937, curr_reward: -2.0, best_reward: 6.0, running_avg_reward: -9.52, curr_epsilon: 0.0064\n",
      "episode_num 938, curr_reward: -16.0, best_reward: 6.0, running_avg_reward: -9.53, curr_epsilon: 0.0063\n",
      "episode_num 939, curr_reward: -9.0, best_reward: 6.0, running_avg_reward: -9.64, curr_epsilon: 0.0063\n",
      "episode_num 940, curr_reward: -14.0, best_reward: 6.0, running_avg_reward: -9.66, curr_epsilon: 0.0063\n",
      "episode_num 941, curr_reward: -8.0, best_reward: 6.0, running_avg_reward: -9.66, curr_epsilon: 0.0063\n",
      "episode_num 942, curr_reward: -17.0, best_reward: 6.0, running_avg_reward: -9.78, curr_epsilon: 0.0063\n",
      "episode_num 943, curr_reward: -18.0, best_reward: 6.0, running_avg_reward: -9.82, curr_epsilon: 0.0063\n",
      "episode_num 944, curr_reward: -16.0, best_reward: 6.0, running_avg_reward: -9.84, curr_epsilon: 0.0063\n",
      "episode_num 945, curr_reward: 3.0, best_reward: 6.0, running_avg_reward: -9.69, curr_epsilon: 0.0063\n",
      "episode_num 946, curr_reward: -6.0, best_reward: 6.0, running_avg_reward: -9.66, curr_epsilon: 0.0062\n",
      "episode_num 947, curr_reward: -11.0, best_reward: 6.0, running_avg_reward: -9.64, curr_epsilon: 0.0062\n",
      "episode_num 948, curr_reward: -10.0, best_reward: 6.0, running_avg_reward: -9.73, curr_epsilon: 0.0062\n",
      "episode_num 949, curr_reward: -4.0, best_reward: 6.0, running_avg_reward: -9.68, curr_epsilon: 0.0062\n",
      "episode_num 950, curr_reward: -1.0, best_reward: 6.0, running_avg_reward: -9.61, curr_epsilon: 0.0062\n",
      "episode_num 951, curr_reward: -5.0, best_reward: 6.0, running_avg_reward: -9.55, curr_epsilon: 0.0062\n",
      "episode_num 952, curr_reward: -10.0, best_reward: 6.0, running_avg_reward: -9.62, curr_epsilon: 0.0062\n",
      "episode_num 953, curr_reward: -2.0, best_reward: 6.0, running_avg_reward: -9.53, curr_epsilon: 0.0062\n",
      "episode_num 954, curr_reward: -16.0, best_reward: 6.0, running_avg_reward: -9.58, curr_epsilon: 0.0061\n",
      "episode_num 955, curr_reward: -8.0, best_reward: 6.0, running_avg_reward: -9.53, curr_epsilon: 0.0061\n",
      "episode_num 956, curr_reward: -6.0, best_reward: 6.0, running_avg_reward: -9.48, curr_epsilon: 0.0061\n",
      "episode_num 957, curr_reward: -17.0, best_reward: 6.0, running_avg_reward: -9.51, curr_epsilon: 0.0061\n",
      "episode_num 958, curr_reward: -15.0, best_reward: 6.0, running_avg_reward: -9.59, curr_epsilon: 0.0061\n",
      "episode_num 959, curr_reward: -12.0, best_reward: 6.0, running_avg_reward: -9.57, curr_epsilon: 0.0061\n",
      "episode_num 960, curr_reward: -3.0, best_reward: 6.0, running_avg_reward: -9.51, curr_epsilon: 0.0061\n",
      "episode_num 961, curr_reward: -12.0, best_reward: 6.0, running_avg_reward: -9.59, curr_epsilon: 0.0061\n",
      "episode_num 962, curr_reward: -1.0, best_reward: 6.0, running_avg_reward: -9.46, curr_epsilon: 0.0061\n",
      "episode_num 963, curr_reward: -1.0, best_reward: 6.0, running_avg_reward: -9.34, curr_epsilon: 0.006\n",
      "episode_num 964, curr_reward: -12.0, best_reward: 6.0, running_avg_reward: -9.34, curr_epsilon: 0.006\n",
      "checkpointing current model weights. highest running_average_reward of -9.21 achieved!\n",
      "episode_num 965, curr_reward: 9.0, best_reward: 9.0, running_avg_reward: -9.21, curr_epsilon: 0.006\n",
      "episode_num 966, curr_reward: -14.0, best_reward: 9.0, running_avg_reward: -9.26, curr_epsilon: 0.006\n",
      "episode_num 967, curr_reward: -17.0, best_reward: 9.0, running_avg_reward: -9.44, curr_epsilon: 0.006\n",
      "episode_num 968, curr_reward: -13.0, best_reward: 9.0, running_avg_reward: -9.53, curr_epsilon: 0.006\n",
      "episode_num 969, curr_reward: -10.0, best_reward: 9.0, running_avg_reward: -9.51, curr_epsilon: 0.006\n",
      "episode_num 970, curr_reward: 7.0, best_reward: 9.0, running_avg_reward: -9.35, curr_epsilon: 0.006\n",
      "episode_num 971, curr_reward: -16.0, best_reward: 9.0, running_avg_reward: -9.48, curr_epsilon: 0.006\n",
      "episode_num 972, curr_reward: -6.0, best_reward: 9.0, running_avg_reward: -9.43, curr_epsilon: 0.006\n",
      "episode_num 973, curr_reward: -12.0, best_reward: 9.0, running_avg_reward: -9.58, curr_epsilon: 0.006\n",
      "episode_num 974, curr_reward: -10.0, best_reward: 9.0, running_avg_reward: -9.6, curr_epsilon: 0.0059\n",
      "episode_num 975, curr_reward: -9.0, best_reward: 9.0, running_avg_reward: -9.64, curr_epsilon: 0.0059\n",
      "episode_num 976, curr_reward: 6.0, best_reward: 9.0, running_avg_reward: -9.38, curr_epsilon: 0.0059\n",
      "episode_num 977, curr_reward: 2.0, best_reward: 9.0, running_avg_reward: -9.33, curr_epsilon: 0.0059\n",
      "checkpointing current model weights. highest running_average_reward of -9.19 achieved!\n",
      "episode_num 978, curr_reward: 1.0, best_reward: 9.0, running_avg_reward: -9.19, curr_epsilon: 0.0059\n",
      "episode_num 979, curr_reward: -14.0, best_reward: 9.0, running_avg_reward: -9.23, curr_epsilon: 0.0059\n",
      "episode_num 980, curr_reward: -13.0, best_reward: 9.0, running_avg_reward: -9.32, curr_epsilon: 0.0059\n",
      "episode_num 981, curr_reward: -11.0, best_reward: 9.0, running_avg_reward: -9.34, curr_epsilon: 0.0059\n",
      "episode_num 982, curr_reward: -14.0, best_reward: 9.0, running_avg_reward: -9.39, curr_epsilon: 0.0059\n",
      "episode_num 983, curr_reward: 5.0, best_reward: 9.0, running_avg_reward: -9.24, curr_epsilon: 0.0059\n",
      "episode_num 984, curr_reward: -4.0, best_reward: 9.0, running_avg_reward: -9.19, curr_epsilon: 0.0058\n",
      "checkpointing current model weights. highest running_average_reward of -9.12 achieved!\n",
      "episode_num 985, curr_reward: -8.0, best_reward: 9.0, running_avg_reward: -9.12, curr_epsilon: 0.0058\n",
      "episode_num 986, curr_reward: -12.0, best_reward: 9.0, running_avg_reward: -9.14, curr_epsilon: 0.0058\n",
      "checkpointing current model weights. highest running_average_reward of -9.09 achieved!\n",
      "episode_num 987, curr_reward: -4.0, best_reward: 9.0, running_avg_reward: -9.09, curr_epsilon: 0.0058\n",
      "checkpointing current model weights. highest running_average_reward of -9.08 achieved!\n",
      "episode_num 988, curr_reward: -12.0, best_reward: 9.0, running_avg_reward: -9.08, curr_epsilon: 0.0058\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 989, curr_reward: -11.0, best_reward: 9.0, running_avg_reward: -9.16, curr_epsilon: 0.0058\n",
      "checkpointing current model weights. highest running_average_reward of -8.97 achieved!\n",
      "episode_num 990, curr_reward: 5.0, best_reward: 9.0, running_avg_reward: -8.97, curr_epsilon: 0.0058\n",
      "checkpointing current model weights. highest running_average_reward of -8.95 achieved!\n",
      "episode_num 991, curr_reward: -4.0, best_reward: 9.0, running_avg_reward: -8.95, curr_epsilon: 0.0058\n",
      "checkpointing current model weights. highest running_average_reward of -8.9 achieved!\n",
      "episode_num 992, curr_reward: -1.0, best_reward: 9.0, running_avg_reward: -8.9, curr_epsilon: 0.0058\n",
      "checkpointing current model weights. highest running_average_reward of -8.89 achieved!\n",
      "episode_num 993, curr_reward: -10.0, best_reward: 9.0, running_avg_reward: -8.89, curr_epsilon: 0.0058\n",
      "episode_num 994, curr_reward: -13.0, best_reward: 9.0, running_avg_reward: -8.89, curr_epsilon: 0.0058\n",
      "episode_num 995, curr_reward: -4.0, best_reward: 9.0, running_avg_reward: -8.89, curr_epsilon: 0.0058\n",
      "checkpointing current model weights. highest running_average_reward of -8.85 achieved!\n",
      "episode_num 996, curr_reward: -9.0, best_reward: 9.0, running_avg_reward: -8.85, curr_epsilon: 0.0057\n",
      "checkpointing current model weights. highest running_average_reward of -8.76 achieved!\n",
      "episode_num 997, curr_reward: -4.0, best_reward: 9.0, running_avg_reward: -8.76, curr_epsilon: 0.0057\n",
      "checkpointing current model weights. highest running_average_reward of -8.69 achieved!\n",
      "episode_num 998, curr_reward: -2.0, best_reward: 9.0, running_avg_reward: -8.69, curr_epsilon: 0.0057\n",
      "checkpointing current model weights. highest running_average_reward of -8.63 achieved!\n",
      "episode_num 999, curr_reward: -5.0, best_reward: 9.0, running_avg_reward: -8.63, curr_epsilon: 0.0057\n",
      "episode_num 1000, curr_reward: -10.0, best_reward: 9.0, running_avg_reward: -8.63, curr_epsilon: 0.0057\n",
      "episode_num 1001, curr_reward: -16.0, best_reward: 9.0, running_avg_reward: -8.66, curr_epsilon: 0.0057\n",
      "episode_num 1002, curr_reward: -11.0, best_reward: 9.0, running_avg_reward: -8.63, curr_epsilon: 0.0057\n",
      "checkpointing current model weights. highest running_average_reward of -8.52 achieved!\n",
      "episode_num 1003, curr_reward: -2.0, best_reward: 9.0, running_avg_reward: -8.52, curr_epsilon: 0.0057\n",
      "checkpointing current model weights. highest running_average_reward of -8.36 achieved!\n",
      "episode_num 1004, curr_reward: 6.0, best_reward: 9.0, running_avg_reward: -8.36, curr_epsilon: 0.0057\n",
      "checkpointing current model weights. highest running_average_reward of -8.3 achieved!\n",
      "episode_num 1005, curr_reward: -10.0, best_reward: 9.0, running_avg_reward: -8.3, curr_epsilon: 0.0057\n",
      "episode_num 1006, curr_reward: -8.0, best_reward: 9.0, running_avg_reward: -8.31, curr_epsilon: 0.0057\n",
      "episode_num 1007, curr_reward: -15.0, best_reward: 9.0, running_avg_reward: -8.33, curr_epsilon: 0.0057\n",
      "episode_num 1008, curr_reward: -13.0, best_reward: 9.0, running_avg_reward: -8.31, curr_epsilon: 0.0057\n",
      "episode_num 1009, curr_reward: -11.0, best_reward: 9.0, running_avg_reward: -8.38, curr_epsilon: 0.0057\n",
      "episode_num 1010, curr_reward: -13.0, best_reward: 9.0, running_avg_reward: -8.41, curr_epsilon: 0.0056\n",
      "episode_num 1011, curr_reward: -7.0, best_reward: 9.0, running_avg_reward: -8.37, curr_epsilon: 0.0056\n",
      "episode_num 1012, curr_reward: -1.0, best_reward: 9.0, running_avg_reward: -8.34, curr_epsilon: 0.0056\n",
      "episode_num 1013, curr_reward: -11.0, best_reward: 9.0, running_avg_reward: -8.37, curr_epsilon: 0.0056\n",
      "episode_num 1014, curr_reward: -11.0, best_reward: 9.0, running_avg_reward: -8.31, curr_epsilon: 0.0056\n",
      "checkpointing current model weights. highest running_average_reward of -8.14 achieved!\n",
      "episode_num 1015, curr_reward: 3.0, best_reward: 9.0, running_avg_reward: -8.14, curr_epsilon: 0.0056\n",
      "episode_num 1016, curr_reward: -7.0, best_reward: 9.0, running_avg_reward: -8.26, curr_epsilon: 0.0056\n",
      "episode_num 1017, curr_reward: -6.0, best_reward: 9.0, running_avg_reward: -8.22, curr_epsilon: 0.0056\n",
      "checkpointing current model weights. highest running_average_reward of -8.1 achieved!\n",
      "episode_num 1018, curr_reward: 3.0, best_reward: 9.0, running_avg_reward: -8.1, curr_epsilon: 0.0056\n",
      "checkpointing current model weights. highest running_average_reward of -8.04 achieved!\n",
      "episode_num 1019, curr_reward: -9.0, best_reward: 9.0, running_avg_reward: -8.04, curr_epsilon: 0.0056\n",
      "checkpointing current model weights. highest running_average_reward of -8.03 achieved!\n",
      "episode_num 1020, curr_reward: -8.0, best_reward: 9.0, running_avg_reward: -8.03, curr_epsilon: 0.0056\n",
      "episode_num 1021, curr_reward: -10.0, best_reward: 9.0, running_avg_reward: -8.07, curr_epsilon: 0.0056\n",
      "checkpointing current model weights. highest running_average_reward of -7.97 achieved!\n",
      "episode_num 1022, curr_reward: -4.0, best_reward: 9.0, running_avg_reward: -7.97, curr_epsilon: 0.0056\n",
      "checkpointing current model weights. highest running_average_reward of -7.88 achieved!\n",
      "episode_num 1023, curr_reward: -8.0, best_reward: 9.0, running_avg_reward: -7.88, curr_epsilon: 0.0056\n",
      "episode_num 1024, curr_reward: -12.0, best_reward: 9.0, running_avg_reward: -7.92, curr_epsilon: 0.0055\n",
      "episode_num 1025, curr_reward: -11.0, best_reward: 9.0, running_avg_reward: -7.88, curr_epsilon: 0.0055\n",
      "checkpointing current model weights. highest running_average_reward of -7.85 achieved!\n",
      "episode_num 1026, curr_reward: -11.0, best_reward: 9.0, running_avg_reward: -7.85, curr_epsilon: 0.0055\n",
      "episode_num 1027, curr_reward: -14.0, best_reward: 9.0, running_avg_reward: -7.87, curr_epsilon: 0.0055\n",
      "episode_num 1028, curr_reward: -15.0, best_reward: 9.0, running_avg_reward: -7.88, curr_epsilon: 0.0055\n",
      "episode_num 1029, curr_reward: -15.0, best_reward: 9.0, running_avg_reward: -7.96, curr_epsilon: 0.0055\n",
      "episode_num 1030, curr_reward: -9.0, best_reward: 9.0, running_avg_reward: -8.0, curr_epsilon: 0.0055\n",
      "episode_num 1031, curr_reward: -7.0, best_reward: 9.0, running_avg_reward: -7.96, curr_epsilon: 0.0055\n",
      "episode_num 1032, curr_reward: -7.0, best_reward: 9.0, running_avg_reward: -7.92, curr_epsilon: 0.0055\n",
      "episode_num 1033, curr_reward: -12.0, best_reward: 9.0, running_avg_reward: -7.94, curr_epsilon: 0.0055\n",
      "episode_num 1034, curr_reward: -9.0, best_reward: 9.0, running_avg_reward: -7.98, curr_epsilon: 0.0055\n",
      "episode_num 1035, curr_reward: -11.0, best_reward: 9.0, running_avg_reward: -7.93, curr_epsilon: 0.0055\n",
      "episode_num 1036, curr_reward: -11.0, best_reward: 9.0, running_avg_reward: -7.99, curr_epsilon: 0.0055\n",
      "episode_num 1037, curr_reward: -13.0, best_reward: 9.0, running_avg_reward: -8.1, curr_epsilon: 0.0055\n",
      "episode_num 1038, curr_reward: -12.0, best_reward: 9.0, running_avg_reward: -8.06, curr_epsilon: 0.0055\n",
      "episode_num 1039, curr_reward: -6.0, best_reward: 9.0, running_avg_reward: -8.03, curr_epsilon: 0.0055\n",
      "episode_num 1040, curr_reward: -10.0, best_reward: 9.0, running_avg_reward: -7.99, curr_epsilon: 0.0055\n",
      "episode_num 1041, curr_reward: -10.0, best_reward: 9.0, running_avg_reward: -8.01, curr_epsilon: 0.0055\n",
      "episode_num 1042, curr_reward: -11.0, best_reward: 9.0, running_avg_reward: -7.95, curr_epsilon: 0.0055\n",
      "episode_num 1043, curr_reward: -14.0, best_reward: 9.0, running_avg_reward: -7.91, curr_epsilon: 0.0055\n",
      "episode_num 1044, curr_reward: -12.0, best_reward: 9.0, running_avg_reward: -7.87, curr_epsilon: 0.0054\n",
      "episode_num 1045, curr_reward: -12.0, best_reward: 9.0, running_avg_reward: -8.02, curr_epsilon: 0.0054\n",
      "episode_num 1046, curr_reward: -6.0, best_reward: 9.0, running_avg_reward: -8.02, curr_epsilon: 0.0054\n",
      "episode_num 1047, curr_reward: -18.0, best_reward: 9.0, running_avg_reward: -8.09, curr_epsilon: 0.0054\n",
      "episode_num 1048, curr_reward: -6.0, best_reward: 9.0, running_avg_reward: -8.05, curr_epsilon: 0.0054\n",
      "episode_num 1049, curr_reward: -8.0, best_reward: 9.0, running_avg_reward: -8.09, curr_epsilon: 0.0054\n",
      "episode_num 1050, curr_reward: -6.0, best_reward: 9.0, running_avg_reward: -8.14, curr_epsilon: 0.0054\n",
      "episode_num 1051, curr_reward: -13.0, best_reward: 9.0, running_avg_reward: -8.22, curr_epsilon: 0.0054\n",
      "episode_num 1052, curr_reward: -19.0, best_reward: 9.0, running_avg_reward: -8.31, curr_epsilon: 0.0054\n",
      "episode_num 1053, curr_reward: 8.0, best_reward: 9.0, running_avg_reward: -8.21, curr_epsilon: 0.0054\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 1054, curr_reward: 5.0, best_reward: 9.0, running_avg_reward: -8.0, curr_epsilon: 0.0054\n",
      "episode_num 1055, curr_reward: -1.0, best_reward: 9.0, running_avg_reward: -7.93, curr_epsilon: 0.0054\n",
      "episode_num 1056, curr_reward: -8.0, best_reward: 9.0, running_avg_reward: -7.95, curr_epsilon: 0.0054\n",
      "checkpointing current model weights. highest running_average_reward of -7.8 achieved!\n",
      "episode_num 1057, curr_reward: -2.0, best_reward: 9.0, running_avg_reward: -7.8, curr_epsilon: 0.0054\n",
      "checkpointing current model weights. highest running_average_reward of -7.77 achieved!\n",
      "episode_num 1058, curr_reward: -12.0, best_reward: 9.0, running_avg_reward: -7.77, curr_epsilon: 0.0054\n",
      "checkpointing current model weights. highest running_average_reward of -7.72 achieved!\n",
      "episode_num 1059, curr_reward: -7.0, best_reward: 9.0, running_avg_reward: -7.72, curr_epsilon: 0.0054\n",
      "episode_num 1060, curr_reward: -7.0, best_reward: 9.0, running_avg_reward: -7.76, curr_epsilon: 0.0054\n",
      "checkpointing current model weights. highest running_average_reward of -7.69 achieved!\n",
      "episode_num 1061, curr_reward: -5.0, best_reward: 9.0, running_avg_reward: -7.69, curr_epsilon: 0.0054\n",
      "episode_num 1062, curr_reward: -15.0, best_reward: 9.0, running_avg_reward: -7.83, curr_epsilon: 0.0054\n",
      "episode_num 1063, curr_reward: -5.0, best_reward: 9.0, running_avg_reward: -7.87, curr_epsilon: 0.0054\n",
      "episode_num 1064, curr_reward: -10.0, best_reward: 9.0, running_avg_reward: -7.85, curr_epsilon: 0.0054\n",
      "episode_num 1065, curr_reward: -11.0, best_reward: 9.0, running_avg_reward: -8.05, curr_epsilon: 0.0054\n",
      "episode_num 1066, curr_reward: -10.0, best_reward: 9.0, running_avg_reward: -8.01, curr_epsilon: 0.0054\n",
      "episode_num 1067, curr_reward: -8.0, best_reward: 9.0, running_avg_reward: -7.92, curr_epsilon: 0.0053\n",
      "episode_num 1068, curr_reward: -13.0, best_reward: 9.0, running_avg_reward: -7.92, curr_epsilon: 0.0053\n",
      "episode_num 1069, curr_reward: -8.0, best_reward: 9.0, running_avg_reward: -7.9, curr_epsilon: 0.0053\n",
      "episode_num 1070, curr_reward: -9.0, best_reward: 9.0, running_avg_reward: -8.06, curr_epsilon: 0.0053\n",
      "episode_num 1071, curr_reward: -11.0, best_reward: 9.0, running_avg_reward: -8.01, curr_epsilon: 0.0053\n",
      "episode_num 1072, curr_reward: -10.0, best_reward: 9.0, running_avg_reward: -8.05, curr_epsilon: 0.0053\n",
      "episode_num 1073, curr_reward: -13.0, best_reward: 9.0, running_avg_reward: -8.06, curr_epsilon: 0.0053\n",
      "episode_num 1074, curr_reward: 10.0, best_reward: 10.0, running_avg_reward: -7.86, curr_epsilon: 0.0053\n",
      "episode_num 1075, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -7.8, curr_epsilon: 0.0053\n",
      "episode_num 1076, curr_reward: -19.0, best_reward: 10.0, running_avg_reward: -8.05, curr_epsilon: 0.0053\n",
      "episode_num 1077, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.18, curr_epsilon: 0.0053\n",
      "episode_num 1078, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.27, curr_epsilon: 0.0053\n",
      "episode_num 1079, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.2, curr_epsilon: 0.0053\n",
      "episode_num 1080, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.18, curr_epsilon: 0.0053\n",
      "episode_num 1081, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.18, curr_epsilon: 0.0053\n",
      "episode_num 1082, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.14, curr_epsilon: 0.0053\n",
      "episode_num 1083, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.27, curr_epsilon: 0.0053\n",
      "episode_num 1084, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.32, curr_epsilon: 0.0053\n",
      "episode_num 1085, curr_reward: -17.0, best_reward: 10.0, running_avg_reward: -8.41, curr_epsilon: 0.0053\n",
      "episode_num 1086, curr_reward: 2.0, best_reward: 10.0, running_avg_reward: -8.27, curr_epsilon: 0.0053\n",
      "episode_num 1087, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.34, curr_epsilon: 0.0053\n",
      "episode_num 1088, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.26, curr_epsilon: 0.0053\n",
      "episode_num 1089, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.2, curr_epsilon: 0.0053\n",
      "episode_num 1090, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.4, curr_epsilon: 0.0053\n",
      "episode_num 1091, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -8.39, curr_epsilon: 0.0053\n",
      "episode_num 1092, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.49, curr_epsilon: 0.0053\n",
      "episode_num 1093, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.47, curr_epsilon: 0.0053\n",
      "episode_num 1094, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.44, curr_epsilon: 0.0053\n",
      "episode_num 1095, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.5, curr_epsilon: 0.0053\n",
      "episode_num 1096, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.52, curr_epsilon: 0.0053\n",
      "episode_num 1097, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.57, curr_epsilon: 0.0053\n",
      "episode_num 1098, curr_reward: -19.0, best_reward: 10.0, running_avg_reward: -8.74, curr_epsilon: 0.0053\n",
      "episode_num 1099, curr_reward: -17.0, best_reward: 10.0, running_avg_reward: -8.86, curr_epsilon: 0.0053\n",
      "episode_num 1100, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -8.79, curr_epsilon: 0.0053\n",
      "episode_num 1101, curr_reward: -17.0, best_reward: 10.0, running_avg_reward: -8.8, curr_epsilon: 0.0053\n",
      "episode_num 1102, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.79, curr_epsilon: 0.0052\n",
      "episode_num 1103, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.85, curr_epsilon: 0.0052\n",
      "episode_num 1104, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -9.06, curr_epsilon: 0.0052\n",
      "episode_num 1105, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.05, curr_epsilon: 0.0052\n",
      "episode_num 1106, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -9.04, curr_epsilon: 0.0052\n",
      "episode_num 1107, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.95, curr_epsilon: 0.0052\n",
      "episode_num 1108, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.92, curr_epsilon: 0.0052\n",
      "episode_num 1109, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.87, curr_epsilon: 0.0052\n",
      "episode_num 1110, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.83, curr_epsilon: 0.0052\n",
      "episode_num 1111, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.86, curr_epsilon: 0.0052\n",
      "episode_num 1112, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.92, curr_epsilon: 0.0052\n",
      "episode_num 1113, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.93, curr_epsilon: 0.0052\n",
      "episode_num 1114, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.92, curr_epsilon: 0.0052\n",
      "episode_num 1115, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -9.0, curr_epsilon: 0.0052\n",
      "episode_num 1116, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -9.05, curr_epsilon: 0.0052\n",
      "episode_num 1117, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -9.06, curr_epsilon: 0.0052\n",
      "episode_num 1118, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -9.15, curr_epsilon: 0.0052\n",
      "episode_num 1119, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.15, curr_epsilon: 0.0052\n",
      "episode_num 1120, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -9.08, curr_epsilon: 0.0052\n",
      "episode_num 1121, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -9.11, curr_epsilon: 0.0052\n",
      "episode_num 1122, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -9.21, curr_epsilon: 0.0052\n",
      "episode_num 1123, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -9.29, curr_epsilon: 0.0052\n",
      "episode_num 1124, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -9.3, curr_epsilon: 0.0052\n",
      "episode_num 1125, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.28, curr_epsilon: 0.0052\n",
      "episode_num 1126, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -9.29, curr_epsilon: 0.0052\n",
      "episode_num 1127, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -9.3, curr_epsilon: 0.0052\n",
      "episode_num 1128, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -9.25, curr_epsilon: 0.0052\n",
      "episode_num 1129, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -9.18, curr_epsilon: 0.0052\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 1130, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -9.15, curr_epsilon: 0.0052\n",
      "episode_num 1131, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -9.09, curr_epsilon: 0.0052\n",
      "episode_num 1132, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -9.06, curr_epsilon: 0.0052\n",
      "episode_num 1133, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -9.05, curr_epsilon: 0.0052\n",
      "episode_num 1134, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.05, curr_epsilon: 0.0052\n",
      "episode_num 1135, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -9.04, curr_epsilon: 0.0052\n",
      "episode_num 1136, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.02, curr_epsilon: 0.0052\n",
      "episode_num 1137, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -9.03, curr_epsilon: 0.0052\n",
      "episode_num 1138, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.99, curr_epsilon: 0.0052\n",
      "episode_num 1139, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -9.06, curr_epsilon: 0.0052\n",
      "episode_num 1140, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.05, curr_epsilon: 0.0052\n",
      "episode_num 1141, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -9.08, curr_epsilon: 0.0052\n",
      "episode_num 1142, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -9.07, curr_epsilon: 0.0052\n",
      "episode_num 1143, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.98, curr_epsilon: 0.0052\n",
      "episode_num 1144, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -9.02, curr_epsilon: 0.0052\n",
      "episode_num 1145, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.98, curr_epsilon: 0.0052\n",
      "episode_num 1146, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -9.04, curr_epsilon: 0.0052\n",
      "episode_num 1147, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.95, curr_epsilon: 0.0052\n",
      "episode_num 1148, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -9.04, curr_epsilon: 0.0052\n",
      "episode_num 1149, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.05, curr_epsilon: 0.0052\n",
      "episode_num 1150, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -9.13, curr_epsilon: 0.0052\n",
      "episode_num 1151, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -9.04, curr_epsilon: 0.0052\n",
      "episode_num 1152, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -9.0, curr_epsilon: 0.0051\n",
      "episode_num 1153, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.17, curr_epsilon: 0.0051\n",
      "episode_num 1154, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -9.32, curr_epsilon: 0.0051\n",
      "episode_num 1155, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -9.45, curr_epsilon: 0.0051\n",
      "episode_num 1156, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -9.44, curr_epsilon: 0.0051\n",
      "episode_num 1157, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -9.53, curr_epsilon: 0.0051\n",
      "episode_num 1158, curr_reward: 3.0, best_reward: 10.0, running_avg_reward: -9.38, curr_epsilon: 0.0051\n",
      "episode_num 1159, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -9.37, curr_epsilon: 0.0051\n",
      "episode_num 1160, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -9.37, curr_epsilon: 0.0051\n",
      "episode_num 1161, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.41, curr_epsilon: 0.0051\n",
      "episode_num 1162, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -9.39, curr_epsilon: 0.0051\n",
      "episode_num 1163, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -9.48, curr_epsilon: 0.0051\n",
      "episode_num 1164, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -9.42, curr_epsilon: 0.0051\n",
      "episode_num 1165, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -9.41, curr_epsilon: 0.0051\n",
      "episode_num 1166, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -9.32, curr_epsilon: 0.0051\n",
      "episode_num 1167, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -9.4, curr_epsilon: 0.0051\n",
      "episode_num 1168, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.36, curr_epsilon: 0.0051\n",
      "episode_num 1169, curr_reward: 1.0, best_reward: 10.0, running_avg_reward: -9.27, curr_epsilon: 0.0051\n",
      "episode_num 1170, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -9.29, curr_epsilon: 0.0051\n",
      "episode_num 1171, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -9.29, curr_epsilon: 0.0051\n",
      "episode_num 1172, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -9.3, curr_epsilon: 0.0051\n",
      "episode_num 1173, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -9.24, curr_epsilon: 0.0051\n",
      "episode_num 1174, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -9.4, curr_epsilon: 0.0051\n",
      "episode_num 1175, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -9.44, curr_epsilon: 0.0051\n",
      "episode_num 1176, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -9.36, curr_epsilon: 0.0051\n",
      "episode_num 1177, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -9.31, curr_epsilon: 0.0051\n",
      "episode_num 1178, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -9.35, curr_epsilon: 0.0051\n",
      "episode_num 1179, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -9.33, curr_epsilon: 0.0051\n",
      "episode_num 1180, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.31, curr_epsilon: 0.0051\n",
      "episode_num 1181, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -9.3, curr_epsilon: 0.0051\n",
      "episode_num 1182, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.29, curr_epsilon: 0.0051\n",
      "episode_num 1183, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -9.22, curr_epsilon: 0.0051\n",
      "episode_num 1184, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -9.24, curr_epsilon: 0.0051\n",
      "episode_num 1185, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -9.08, curr_epsilon: 0.0051\n",
      "episode_num 1186, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -9.14, curr_epsilon: 0.0051\n",
      "episode_num 1187, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -9.14, curr_epsilon: 0.0051\n",
      "episode_num 1188, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -9.22, curr_epsilon: 0.0051\n",
      "episode_num 1189, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -9.29, curr_epsilon: 0.0051\n",
      "episode_num 1190, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -9.21, curr_epsilon: 0.0051\n",
      "episode_num 1191, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -9.3, curr_epsilon: 0.0051\n",
      "episode_num 1192, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.28, curr_epsilon: 0.0051\n",
      "episode_num 1193, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -9.23, curr_epsilon: 0.0051\n",
      "episode_num 1194, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -9.23, curr_epsilon: 0.0051\n",
      "episode_num 1195, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -9.21, curr_epsilon: 0.0051\n",
      "episode_num 1196, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -9.2, curr_epsilon: 0.0051\n",
      "episode_num 1197, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -9.18, curr_epsilon: 0.0051\n",
      "episode_num 1198, curr_reward: 6.0, best_reward: 10.0, running_avg_reward: -8.93, curr_epsilon: 0.0051\n",
      "episode_num 1199, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -8.78, curr_epsilon: 0.0051\n",
      "episode_num 1200, curr_reward: 6.0, best_reward: 10.0, running_avg_reward: -8.69, curr_epsilon: 0.0051\n",
      "episode_num 1201, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -8.54, curr_epsilon: 0.0051\n",
      "episode_num 1202, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.48, curr_epsilon: 0.0051\n",
      "episode_num 1203, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -8.56, curr_epsilon: 0.0051\n",
      "episode_num 1204, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -8.42, curr_epsilon: 0.0051\n",
      "episode_num 1205, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.47, curr_epsilon: 0.0051\n",
      "episode_num 1206, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.44, curr_epsilon: 0.0051\n",
      "episode_num 1207, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.42, curr_epsilon: 0.0051\n",
      "episode_num 1208, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.4, curr_epsilon: 0.0051\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 1209, curr_reward: 1.0, best_reward: 10.0, running_avg_reward: -8.33, curr_epsilon: 0.0051\n",
      "episode_num 1210, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.37, curr_epsilon: 0.0051\n",
      "episode_num 1211, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.34, curr_epsilon: 0.0051\n",
      "episode_num 1212, curr_reward: 1.0, best_reward: 10.0, running_avg_reward: -8.26, curr_epsilon: 0.0051\n",
      "episode_num 1213, curr_reward: 1.0, best_reward: 10.0, running_avg_reward: -8.13, curr_epsilon: 0.0051\n",
      "episode_num 1214, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.12, curr_epsilon: 0.0051\n",
      "episode_num 1215, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.17, curr_epsilon: 0.0051\n",
      "episode_num 1216, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -8.08, curr_epsilon: 0.0051\n",
      "episode_num 1217, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.16, curr_epsilon: 0.0051\n",
      "episode_num 1218, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.21, curr_epsilon: 0.0051\n",
      "episode_num 1219, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.21, curr_epsilon: 0.0051\n",
      "episode_num 1220, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.3, curr_epsilon: 0.0051\n",
      "episode_num 1221, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.25, curr_epsilon: 0.0051\n",
      "episode_num 1222, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.23, curr_epsilon: 0.0051\n",
      "episode_num 1223, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.19, curr_epsilon: 0.0051\n",
      "episode_num 1224, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.15, curr_epsilon: 0.0051\n",
      "episode_num 1225, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.13, curr_epsilon: 0.0051\n",
      "episode_num 1226, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.09, curr_epsilon: 0.0051\n",
      "episode_num 1227, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.98, curr_epsilon: 0.0051\n",
      "episode_num 1228, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -7.99, curr_epsilon: 0.0051\n",
      "episode_num 1229, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.98, curr_epsilon: 0.0051\n",
      "episode_num 1230, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.07, curr_epsilon: 0.0051\n",
      "episode_num 1231, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.19, curr_epsilon: 0.0051\n",
      "episode_num 1232, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.25, curr_epsilon: 0.0051\n",
      "episode_num 1233, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -8.3, curr_epsilon: 0.0051\n",
      "episode_num 1234, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.3, curr_epsilon: 0.0051\n",
      "episode_num 1235, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.29, curr_epsilon: 0.0051\n",
      "episode_num 1236, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.33, curr_epsilon: 0.0051\n",
      "episode_num 1237, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.29, curr_epsilon: 0.0051\n",
      "episode_num 1238, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.25, curr_epsilon: 0.0051\n",
      "episode_num 1239, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.27, curr_epsilon: 0.0051\n",
      "episode_num 1240, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.3, curr_epsilon: 0.0051\n",
      "episode_num 1241, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -8.33, curr_epsilon: 0.0051\n",
      "episode_num 1242, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.33, curr_epsilon: 0.0051\n",
      "episode_num 1243, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.39, curr_epsilon: 0.0051\n",
      "episode_num 1244, curr_reward: 6.0, best_reward: 10.0, running_avg_reward: -8.17, curr_epsilon: 0.0051\n",
      "episode_num 1245, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.17, curr_epsilon: 0.0051\n",
      "episode_num 1246, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.12, curr_epsilon: 0.0051\n",
      "episode_num 1247, curr_reward: -17.0, best_reward: 10.0, running_avg_reward: -8.2, curr_epsilon: 0.0051\n",
      "episode_num 1248, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.16, curr_epsilon: 0.0051\n",
      "episode_num 1249, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.14, curr_epsilon: 0.0051\n",
      "episode_num 1250, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.09, curr_epsilon: 0.0051\n",
      "episode_num 1251, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -8.21, curr_epsilon: 0.0051\n",
      "episode_num 1252, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.13, curr_epsilon: 0.0051\n",
      "episode_num 1253, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.08, curr_epsilon: 0.0051\n",
      "episode_num 1254, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.1, curr_epsilon: 0.0051\n",
      "episode_num 1255, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.08, curr_epsilon: 0.0051\n",
      "episode_num 1256, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.12, curr_epsilon: 0.005\n",
      "episode_num 1257, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.07, curr_epsilon: 0.005\n",
      "episode_num 1258, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.2, curr_epsilon: 0.005\n",
      "episode_num 1259, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.24, curr_epsilon: 0.005\n",
      "episode_num 1260, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.26, curr_epsilon: 0.005\n",
      "episode_num 1261, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.26, curr_epsilon: 0.005\n",
      "episode_num 1262, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.21, curr_epsilon: 0.005\n",
      "episode_num 1263, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.21, curr_epsilon: 0.005\n",
      "episode_num 1264, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.23, curr_epsilon: 0.005\n",
      "episode_num 1265, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -8.15, curr_epsilon: 0.005\n",
      "episode_num 1266, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.24, curr_epsilon: 0.005\n",
      "episode_num 1267, curr_reward: 8.0, best_reward: 10.0, running_avg_reward: -8.0, curr_epsilon: 0.005\n",
      "episode_num 1268, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.05, curr_epsilon: 0.005\n",
      "episode_num 1269, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -8.09, curr_epsilon: 0.005\n",
      "episode_num 1270, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.07, curr_epsilon: 0.005\n",
      "episode_num 1271, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.03, curr_epsilon: 0.005\n",
      "episode_num 1272, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.0, curr_epsilon: 0.005\n",
      "episode_num 1273, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.03, curr_epsilon: 0.005\n",
      "episode_num 1274, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.08, curr_epsilon: 0.005\n",
      "episode_num 1275, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.12, curr_epsilon: 0.005\n",
      "episode_num 1276, curr_reward: 3.0, best_reward: 10.0, running_avg_reward: -7.98, curr_epsilon: 0.005\n",
      "episode_num 1277, curr_reward: -18.0, best_reward: 10.0, running_avg_reward: -8.1, curr_epsilon: 0.005\n",
      "episode_num 1278, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.12, curr_epsilon: 0.005\n",
      "episode_num 1279, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -8.1, curr_epsilon: 0.005\n",
      "episode_num 1280, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.13, curr_epsilon: 0.005\n",
      "episode_num 1281, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.07, curr_epsilon: 0.005\n",
      "episode_num 1282, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.07, curr_epsilon: 0.005\n",
      "episode_num 1283, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.14, curr_epsilon: 0.005\n",
      "episode_num 1284, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.14, curr_epsilon: 0.005\n",
      "episode_num 1285, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.25, curr_epsilon: 0.005\n",
      "episode_num 1286, curr_reward: 5.0, best_reward: 10.0, running_avg_reward: -8.16, curr_epsilon: 0.005\n",
      "episode_num 1287, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.13, curr_epsilon: 0.005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 1288, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.06, curr_epsilon: 0.005\n",
      "episode_num 1289, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.04, curr_epsilon: 0.005\n",
      "episode_num 1290, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.09, curr_epsilon: 0.005\n",
      "episode_num 1291, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.04, curr_epsilon: 0.005\n",
      "episode_num 1292, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.07, curr_epsilon: 0.005\n",
      "episode_num 1293, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.12, curr_epsilon: 0.005\n",
      "episode_num 1294, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.12, curr_epsilon: 0.005\n",
      "episode_num 1295, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.19, curr_epsilon: 0.005\n",
      "episode_num 1296, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.18, curr_epsilon: 0.005\n",
      "episode_num 1297, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.18, curr_epsilon: 0.005\n",
      "episode_num 1298, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.32, curr_epsilon: 0.005\n",
      "episode_num 1299, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.34, curr_epsilon: 0.005\n",
      "episode_num 1300, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.51, curr_epsilon: 0.005\n",
      "episode_num 1301, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.53, curr_epsilon: 0.005\n",
      "episode_num 1302, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.61, curr_epsilon: 0.005\n",
      "episode_num 1303, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.52, curr_epsilon: 0.005\n",
      "episode_num 1304, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.56, curr_epsilon: 0.005\n",
      "episode_num 1305, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.47, curr_epsilon: 0.005\n",
      "episode_num 1306, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -8.44, curr_epsilon: 0.005\n",
      "episode_num 1307, curr_reward: -17.0, best_reward: 10.0, running_avg_reward: -8.57, curr_epsilon: 0.005\n",
      "episode_num 1308, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.62, curr_epsilon: 0.005\n",
      "episode_num 1309, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.69, curr_epsilon: 0.005\n",
      "episode_num 1310, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.62, curr_epsilon: 0.005\n",
      "episode_num 1311, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.59, curr_epsilon: 0.005\n",
      "episode_num 1312, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -8.76, curr_epsilon: 0.005\n",
      "episode_num 1313, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.86, curr_epsilon: 0.005\n",
      "episode_num 1314, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -8.93, curr_epsilon: 0.005\n",
      "episode_num 1315, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.95, curr_epsilon: 0.005\n",
      "episode_num 1316, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -9.02, curr_epsilon: 0.005\n",
      "episode_num 1317, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.96, curr_epsilon: 0.005\n",
      "episode_num 1318, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.92, curr_epsilon: 0.005\n",
      "episode_num 1319, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.89, curr_epsilon: 0.005\n",
      "episode_num 1320, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.91, curr_epsilon: 0.005\n",
      "episode_num 1321, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.96, curr_epsilon: 0.005\n",
      "episode_num 1322, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.9, curr_epsilon: 0.005\n",
      "episode_num 1323, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -8.79, curr_epsilon: 0.005\n",
      "episode_num 1324, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.82, curr_epsilon: 0.005\n",
      "episode_num 1325, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -8.78, curr_epsilon: 0.005\n",
      "episode_num 1326, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.85, curr_epsilon: 0.005\n",
      "episode_num 1327, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.95, curr_epsilon: 0.005\n",
      "episode_num 1328, curr_reward: 8.0, best_reward: 10.0, running_avg_reward: -8.76, curr_epsilon: 0.005\n",
      "episode_num 1329, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.79, curr_epsilon: 0.005\n",
      "episode_num 1330, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.68, curr_epsilon: 0.005\n",
      "episode_num 1331, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.59, curr_epsilon: 0.005\n",
      "episode_num 1332, curr_reward: -18.0, best_reward: 10.0, running_avg_reward: -8.67, curr_epsilon: 0.005\n",
      "episode_num 1333, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.56, curr_epsilon: 0.005\n",
      "episode_num 1334, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.57, curr_epsilon: 0.005\n",
      "episode_num 1335, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.56, curr_epsilon: 0.005\n",
      "episode_num 1336, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.54, curr_epsilon: 0.005\n",
      "episode_num 1337, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -8.46, curr_epsilon: 0.005\n",
      "episode_num 1338, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.52, curr_epsilon: 0.005\n",
      "episode_num 1339, curr_reward: 2.0, best_reward: 10.0, running_avg_reward: -8.35, curr_epsilon: 0.005\n",
      "episode_num 1340, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.28, curr_epsilon: 0.005\n",
      "episode_num 1341, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.24, curr_epsilon: 0.005\n",
      "episode_num 1342, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.2, curr_epsilon: 0.005\n",
      "episode_num 1343, curr_reward: -18.0, best_reward: 10.0, running_avg_reward: -8.27, curr_epsilon: 0.005\n",
      "episode_num 1344, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.48, curr_epsilon: 0.005\n",
      "episode_num 1345, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.44, curr_epsilon: 0.005\n",
      "episode_num 1346, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.52, curr_epsilon: 0.005\n",
      "episode_num 1347, curr_reward: 6.0, best_reward: 10.0, running_avg_reward: -8.29, curr_epsilon: 0.005\n",
      "episode_num 1348, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.29, curr_epsilon: 0.005\n",
      "episode_num 1349, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.34, curr_epsilon: 0.005\n",
      "episode_num 1350, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.4, curr_epsilon: 0.005\n",
      "episode_num 1351, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.33, curr_epsilon: 0.005\n",
      "episode_num 1352, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.37, curr_epsilon: 0.005\n",
      "episode_num 1353, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.43, curr_epsilon: 0.005\n",
      "episode_num 1354, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.41, curr_epsilon: 0.005\n",
      "episode_num 1355, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.38, curr_epsilon: 0.005\n",
      "episode_num 1356, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.33, curr_epsilon: 0.005\n",
      "episode_num 1357, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.38, curr_epsilon: 0.005\n",
      "episode_num 1358, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.34, curr_epsilon: 0.005\n",
      "episode_num 1359, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.34, curr_epsilon: 0.005\n",
      "episode_num 1360, curr_reward: 8.0, best_reward: 10.0, running_avg_reward: -8.17, curr_epsilon: 0.005\n",
      "episode_num 1361, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.17, curr_epsilon: 0.005\n",
      "episode_num 1362, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.22, curr_epsilon: 0.005\n",
      "episode_num 1363, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.17, curr_epsilon: 0.005\n",
      "episode_num 1364, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.25, curr_epsilon: 0.005\n",
      "episode_num 1365, curr_reward: -18.0, best_reward: 10.0, running_avg_reward: -8.41, curr_epsilon: 0.005\n",
      "episode_num 1366, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.44, curr_epsilon: 0.005\n",
      "episode_num 1367, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.65, curr_epsilon: 0.005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 1368, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.63, curr_epsilon: 0.005\n",
      "episode_num 1369, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.75, curr_epsilon: 0.005\n",
      "episode_num 1370, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -8.68, curr_epsilon: 0.005\n",
      "episode_num 1371, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.76, curr_epsilon: 0.005\n",
      "episode_num 1372, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.72, curr_epsilon: 0.005\n",
      "episode_num 1373, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.76, curr_epsilon: 0.005\n",
      "episode_num 1374, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -8.68, curr_epsilon: 0.005\n",
      "episode_num 1375, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.69, curr_epsilon: 0.005\n",
      "episode_num 1376, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -8.73, curr_epsilon: 0.005\n",
      "episode_num 1377, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.65, curr_epsilon: 0.005\n",
      "episode_num 1378, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -8.54, curr_epsilon: 0.005\n",
      "episode_num 1379, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.64, curr_epsilon: 0.005\n",
      "episode_num 1380, curr_reward: 1.0, best_reward: 10.0, running_avg_reward: -8.51, curr_epsilon: 0.005\n",
      "episode_num 1381, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.54, curr_epsilon: 0.005\n",
      "episode_num 1382, curr_reward: 3.0, best_reward: 10.0, running_avg_reward: -8.42, curr_epsilon: 0.005\n",
      "episode_num 1383, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.39, curr_epsilon: 0.005\n",
      "episode_num 1384, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -8.44, curr_epsilon: 0.005\n",
      "episode_num 1385, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.42, curr_epsilon: 0.005\n",
      "episode_num 1386, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.58, curr_epsilon: 0.005\n",
      "episode_num 1387, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.55, curr_epsilon: 0.005\n",
      "episode_num 1388, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.54, curr_epsilon: 0.005\n",
      "episode_num 1389, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.48, curr_epsilon: 0.005\n",
      "episode_num 1390, curr_reward: -18.0, best_reward: 10.0, running_avg_reward: -8.54, curr_epsilon: 0.005\n",
      "episode_num 1391, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -8.49, curr_epsilon: 0.005\n",
      "episode_num 1392, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.46, curr_epsilon: 0.005\n",
      "episode_num 1393, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.48, curr_epsilon: 0.005\n",
      "episode_num 1394, curr_reward: 8.0, best_reward: 10.0, running_avg_reward: -8.3, curr_epsilon: 0.005\n",
      "episode_num 1395, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.26, curr_epsilon: 0.005\n",
      "episode_num 1396, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.32, curr_epsilon: 0.005\n",
      "episode_num 1397, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.31, curr_epsilon: 0.005\n",
      "episode_num 1398, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.29, curr_epsilon: 0.005\n",
      "episode_num 1399, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.37, curr_epsilon: 0.005\n",
      "episode_num 1400, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.31, curr_epsilon: 0.005\n",
      "episode_num 1401, curr_reward: 1.0, best_reward: 10.0, running_avg_reward: -8.26, curr_epsilon: 0.005\n",
      "episode_num 1402, curr_reward: 4.0, best_reward: 10.0, running_avg_reward: -8.1, curr_epsilon: 0.005\n",
      "episode_num 1403, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -8.04, curr_epsilon: 0.005\n",
      "episode_num 1404, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.09, curr_epsilon: 0.005\n",
      "episode_num 1405, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.16, curr_epsilon: 0.005\n",
      "episode_num 1406, curr_reward: 9.0, best_reward: 10.0, running_avg_reward: -8.06, curr_epsilon: 0.005\n",
      "episode_num 1407, curr_reward: -19.0, best_reward: 10.0, running_avg_reward: -8.08, curr_epsilon: 0.005\n",
      "episode_num 1408, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.02, curr_epsilon: 0.005\n",
      "episode_num 1409, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.03, curr_epsilon: 0.005\n",
      "episode_num 1410, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.04, curr_epsilon: 0.005\n",
      "episode_num 1411, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.04, curr_epsilon: 0.005\n",
      "episode_num 1412, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.0, curr_epsilon: 0.005\n",
      "episode_num 1413, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.0, curr_epsilon: 0.005\n",
      "episode_num 1414, curr_reward: 8.0, best_reward: 10.0, running_avg_reward: -7.76, curr_epsilon: 0.005\n",
      "episode_num 1415, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -7.69, curr_epsilon: 0.005\n",
      "episode_num 1416, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -7.7, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -7.62 achieved!\n",
      "episode_num 1417, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -7.62, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -7.61 achieved!\n",
      "episode_num 1418, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -7.61, curr_epsilon: 0.005\n",
      "episode_num 1419, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -7.68, curr_epsilon: 0.005\n",
      "episode_num 1420, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.64, curr_epsilon: 0.005\n",
      "episode_num 1421, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -7.62, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -7.54 achieved!\n",
      "episode_num 1422, curr_reward: 2.0, best_reward: 10.0, running_avg_reward: -7.54, curr_epsilon: 0.005\n",
      "episode_num 1423, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -7.54, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -7.44 achieved!\n",
      "episode_num 1424, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -7.44, curr_epsilon: 0.005\n",
      "episode_num 1425, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.53, curr_epsilon: 0.005\n",
      "episode_num 1426, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.45, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -7.33 achieved!\n",
      "episode_num 1427, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -7.33, curr_epsilon: 0.005\n",
      "episode_num 1428, curr_reward: 5.0, best_reward: 10.0, running_avg_reward: -7.36, curr_epsilon: 0.005\n",
      "episode_num 1429, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.34, curr_epsilon: 0.005\n",
      "episode_num 1430, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.38, curr_epsilon: 0.005\n",
      "episode_num 1431, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -7.45, curr_epsilon: 0.005\n",
      "episode_num 1432, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -7.33, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -7.3 achieved!\n",
      "episode_num 1433, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -7.3, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -7.27 achieved!\n",
      "episode_num 1434, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.27, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -7.25 achieved!\n",
      "episode_num 1435, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -7.25, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -7.12 achieved!\n",
      "episode_num 1436, curr_reward: 2.0, best_reward: 10.0, running_avg_reward: -7.12, curr_epsilon: 0.005\n",
      "episode_num 1437, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -7.2, curr_epsilon: 0.005\n",
      "episode_num 1438, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -7.16, curr_epsilon: 0.005\n",
      "episode_num 1439, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -7.2, curr_epsilon: 0.005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 1440, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.22, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -7.11 achieved!\n",
      "episode_num 1441, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -7.11, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -7.09 achieved!\n",
      "episode_num 1442, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.09, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -7.07 achieved!\n",
      "episode_num 1443, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -7.07, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -7.01 achieved!\n",
      "episode_num 1444, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.01, curr_epsilon: 0.005\n",
      "episode_num 1445, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.09, curr_epsilon: 0.005\n",
      "episode_num 1446, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.02, curr_epsilon: 0.005\n",
      "episode_num 1447, curr_reward: 1.0, best_reward: 10.0, running_avg_reward: -7.07, curr_epsilon: 0.005\n",
      "episode_num 1448, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.05, curr_epsilon: 0.005\n",
      "episode_num 1449, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.05, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.88 achieved!\n",
      "episode_num 1450, curr_reward: 2.0, best_reward: 10.0, running_avg_reward: -6.88, curr_epsilon: 0.005\n",
      "episode_num 1451, curr_reward: -17.0, best_reward: 10.0, running_avg_reward: -6.96, curr_epsilon: 0.005\n",
      "episode_num 1452, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -6.88, curr_epsilon: 0.005\n",
      "episode_num 1453, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -6.91, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.82 achieved!\n",
      "episode_num 1454, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -6.82, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.72 achieved!\n",
      "episode_num 1455, curr_reward: 1.0, best_reward: 10.0, running_avg_reward: -6.72, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.7 achieved!\n",
      "episode_num 1456, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -6.7, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.69 achieved!\n",
      "episode_num 1457, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -6.69, curr_epsilon: 0.005\n",
      "episode_num 1458, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -6.72, curr_epsilon: 0.005\n",
      "episode_num 1459, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -6.74, curr_epsilon: 0.005\n",
      "episode_num 1460, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -6.98, curr_epsilon: 0.005\n",
      "episode_num 1461, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -7.04, curr_epsilon: 0.005\n",
      "episode_num 1462, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -7.02, curr_epsilon: 0.005\n",
      "episode_num 1463, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.02, curr_epsilon: 0.005\n",
      "episode_num 1464, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.0, curr_epsilon: 0.005\n",
      "episode_num 1465, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -6.94, curr_epsilon: 0.005\n",
      "episode_num 1466, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -6.9, curr_epsilon: 0.005\n",
      "episode_num 1467, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -6.82, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.67 achieved!\n",
      "episode_num 1468, curr_reward: 3.0, best_reward: 10.0, running_avg_reward: -6.67, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.64 achieved!\n",
      "episode_num 1469, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -6.64, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.63 achieved!\n",
      "episode_num 1470, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -6.63, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.57 achieved!\n",
      "episode_num 1471, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -6.57, curr_epsilon: 0.005\n",
      "episode_num 1472, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -6.67, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.47 achieved!\n",
      "episode_num 1473, curr_reward: 6.0, best_reward: 10.0, running_avg_reward: -6.47, curr_epsilon: 0.005\n",
      "episode_num 1474, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -6.49, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.33 achieved!\n",
      "episode_num 1475, curr_reward: 4.0, best_reward: 10.0, running_avg_reward: -6.33, curr_epsilon: 0.005\n",
      "episode_num 1476, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -6.38, curr_epsilon: 0.005\n",
      "episode_num 1477, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -6.36, curr_epsilon: 0.005\n",
      "episode_num 1478, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -6.42, curr_epsilon: 0.005\n",
      "episode_num 1479, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -6.4, curr_epsilon: 0.005\n",
      "episode_num 1480, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -6.45, curr_epsilon: 0.005\n",
      "episode_num 1481, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -6.43, curr_epsilon: 0.005\n",
      "episode_num 1482, curr_reward: 2.0, best_reward: 10.0, running_avg_reward: -6.44, curr_epsilon: 0.005\n",
      "episode_num 1483, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -6.45, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.27 achieved!\n",
      "episode_num 1484, curr_reward: 2.0, best_reward: 10.0, running_avg_reward: -6.27, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.12 achieved!\n",
      "episode_num 1485, curr_reward: 5.0, best_reward: 10.0, running_avg_reward: -6.12, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.09 achieved!\n",
      "episode_num 1486, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -6.09, curr_epsilon: 0.005\n",
      "episode_num 1487, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -6.12, curr_epsilon: 0.005\n",
      "episode_num 1488, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -6.11, curr_epsilon: 0.005\n",
      "episode_num 1489, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -6.15, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.03 achieved!\n",
      "episode_num 1490, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -6.03, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -6.0 achieved!\n",
      "episode_num 1491, curr_reward: 1.0, best_reward: 10.0, running_avg_reward: -6.0, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -5.93 achieved!\n",
      "episode_num 1492, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -5.93, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -5.86 achieved!\n",
      "episode_num 1493, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -5.86, curr_epsilon: 0.005\n",
      "episode_num 1494, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -6.04, curr_epsilon: 0.005\n",
      "episode_num 1495, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -5.95, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -5.82 achieved!\n",
      "episode_num 1496, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -5.82, curr_epsilon: 0.005\n",
      "episode_num 1497, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -5.82, curr_epsilon: 0.005\n",
      "episode_num 1498, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -5.88, curr_epsilon: 0.005\n",
      "checkpointing current model weights. highest running_average_reward of -5.77 achieved!\n",
      "episode_num 1499, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -5.77, curr_epsilon: 0.005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpointing current model weights. highest running_average_reward of -5.7 achieved!\n",
      "episode_num 1500, curr_reward: 2.0, best_reward: 10.0, running_avg_reward: -5.7, curr_epsilon: 0.005\n",
      "episode_num 1501, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -5.87, curr_epsilon: 0.005\n",
      "episode_num 1502, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -5.96, curr_epsilon: 0.005\n",
      "episode_num 1503, curr_reward: 2.0, best_reward: 10.0, running_avg_reward: -5.93, curr_epsilon: 0.005\n",
      "episode_num 1504, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -5.92, curr_epsilon: 0.005\n",
      "episode_num 1505, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -5.93, curr_epsilon: 0.005\n",
      "episode_num 1506, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -6.15, curr_epsilon: 0.005\n",
      "episode_num 1507, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -6.0, curr_epsilon: 0.005\n",
      "episode_num 1508, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -6.01, curr_epsilon: 0.005\n",
      "episode_num 1509, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -5.99, curr_epsilon: 0.005\n",
      "episode_num 1510, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -6.02, curr_epsilon: 0.005\n",
      "episode_num 1511, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -6.03, curr_epsilon: 0.005\n",
      "episode_num 1512, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -6.03, curr_epsilon: 0.005\n",
      "episode_num 1513, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -6.0, curr_epsilon: 0.005\n",
      "episode_num 1514, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -6.18, curr_epsilon: 0.005\n",
      "episode_num 1515, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -6.16, curr_epsilon: 0.005\n",
      "episode_num 1516, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -6.1, curr_epsilon: 0.005\n",
      "episode_num 1517, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -6.15, curr_epsilon: 0.005\n",
      "episode_num 1518, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -6.17, curr_epsilon: 0.005\n",
      "episode_num 1519, curr_reward: 7.0, best_reward: 10.0, running_avg_reward: -5.97, curr_epsilon: 0.005\n",
      "episode_num 1520, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -5.96, curr_epsilon: 0.005\n",
      "episode_num 1521, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -5.97, curr_epsilon: 0.005\n",
      "episode_num 1522, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -6.07, curr_epsilon: 0.005\n",
      "episode_num 1523, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -6.11, curr_epsilon: 0.005\n",
      "episode_num 1524, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -6.13, curr_epsilon: 0.005\n",
      "episode_num 1525, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -6.07, curr_epsilon: 0.005\n",
      "episode_num 1526, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -6.06, curr_epsilon: 0.005\n",
      "episode_num 1527, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -6.09, curr_epsilon: 0.005\n",
      "episode_num 1528, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -6.22, curr_epsilon: 0.005\n",
      "episode_num 1529, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -6.22, curr_epsilon: 0.005\n",
      "episode_num 1530, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -6.19, curr_epsilon: 0.005\n",
      "episode_num 1531, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -6.19, curr_epsilon: 0.005\n",
      "episode_num 1532, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -6.24, curr_epsilon: 0.005\n",
      "episode_num 1533, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -6.23, curr_epsilon: 0.005\n",
      "episode_num 1534, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -6.17, curr_epsilon: 0.005\n",
      "episode_num 1535, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -6.23, curr_epsilon: 0.005\n",
      "episode_num 1536, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -6.28, curr_epsilon: 0.005\n",
      "episode_num 1537, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -6.26, curr_epsilon: 0.005\n",
      "episode_num 1538, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -6.25, curr_epsilon: 0.005\n",
      "episode_num 1539, curr_reward: -18.0, best_reward: 10.0, running_avg_reward: -6.41, curr_epsilon: 0.005\n",
      "episode_num 1540, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -6.43, curr_epsilon: 0.005\n",
      "episode_num 1541, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -6.51, curr_epsilon: 0.005\n",
      "episode_num 1542, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -6.6, curr_epsilon: 0.005\n",
      "episode_num 1543, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -6.5, curr_epsilon: 0.005\n",
      "episode_num 1544, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -6.48, curr_epsilon: 0.005\n",
      "episode_num 1545, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -6.48, curr_epsilon: 0.005\n",
      "episode_num 1546, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -6.47, curr_epsilon: 0.005\n",
      "episode_num 1547, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -6.54, curr_epsilon: 0.005\n",
      "episode_num 1548, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -6.56, curr_epsilon: 0.005\n",
      "episode_num 1549, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -6.56, curr_epsilon: 0.005\n",
      "episode_num 1550, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -6.71, curr_epsilon: 0.005\n",
      "episode_num 1551, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -6.59, curr_epsilon: 0.005\n",
      "episode_num 1552, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -6.7, curr_epsilon: 0.005\n",
      "episode_num 1553, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -6.67, curr_epsilon: 0.005\n",
      "episode_num 1554, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -6.74, curr_epsilon: 0.005\n",
      "episode_num 1555, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -6.87, curr_epsilon: 0.005\n",
      "episode_num 1556, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -6.84, curr_epsilon: 0.005\n",
      "episode_num 1557, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -6.9, curr_epsilon: 0.005\n",
      "episode_num 1558, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -6.94, curr_epsilon: 0.005\n",
      "episode_num 1559, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -6.87, curr_epsilon: 0.005\n",
      "episode_num 1560, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -6.85, curr_epsilon: 0.005\n",
      "episode_num 1561, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -6.79, curr_epsilon: 0.005\n",
      "episode_num 1562, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -6.71, curr_epsilon: 0.005\n",
      "episode_num 1563, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -6.73, curr_epsilon: 0.005\n",
      "episode_num 1564, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -6.62, curr_epsilon: 0.005\n",
      "episode_num 1565, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -6.52, curr_epsilon: 0.005\n",
      "episode_num 1566, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -6.51, curr_epsilon: 0.005\n",
      "episode_num 1567, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -6.58, curr_epsilon: 0.005\n",
      "episode_num 1568, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -6.73, curr_epsilon: 0.005\n",
      "episode_num 1569, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -6.64, curr_epsilon: 0.005\n",
      "episode_num 1570, curr_reward: 6.0, best_reward: 10.0, running_avg_reward: -6.57, curr_epsilon: 0.005\n",
      "episode_num 1571, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -6.52, curr_epsilon: 0.005\n",
      "episode_num 1572, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -6.45, curr_epsilon: 0.005\n",
      "episode_num 1573, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -6.6, curr_epsilon: 0.005\n",
      "episode_num 1574, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -6.7, curr_epsilon: 0.005\n",
      "episode_num 1575, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -6.83, curr_epsilon: 0.005\n",
      "episode_num 1576, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -6.87, curr_epsilon: 0.005\n",
      "episode_num 1577, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -6.87, curr_epsilon: 0.005\n",
      "episode_num 1578, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -6.84, curr_epsilon: 0.005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 1579, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -6.87, curr_epsilon: 0.005\n",
      "episode_num 1580, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -6.94, curr_epsilon: 0.005\n",
      "episode_num 1581, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.01, curr_epsilon: 0.005\n",
      "episode_num 1582, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -7.13, curr_epsilon: 0.005\n",
      "episode_num 1583, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -7.22, curr_epsilon: 0.005\n",
      "episode_num 1584, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.36, curr_epsilon: 0.005\n",
      "episode_num 1585, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.5, curr_epsilon: 0.005\n",
      "episode_num 1586, curr_reward: 3.0, best_reward: 10.0, running_avg_reward: -7.39, curr_epsilon: 0.005\n",
      "episode_num 1587, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -7.37, curr_epsilon: 0.005\n",
      "episode_num 1588, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -7.45, curr_epsilon: 0.005\n",
      "episode_num 1589, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.46, curr_epsilon: 0.005\n",
      "episode_num 1590, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.52, curr_epsilon: 0.005\n",
      "episode_num 1591, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -7.63, curr_epsilon: 0.005\n",
      "episode_num 1592, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.73, curr_epsilon: 0.005\n",
      "episode_num 1593, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.77, curr_epsilon: 0.005\n",
      "episode_num 1594, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -7.8, curr_epsilon: 0.005\n",
      "episode_num 1595, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -7.83, curr_epsilon: 0.005\n",
      "episode_num 1596, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -7.96, curr_epsilon: 0.005\n",
      "episode_num 1597, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.01, curr_epsilon: 0.005\n",
      "episode_num 1598, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.0, curr_epsilon: 0.005\n",
      "episode_num 1599, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -8.15, curr_epsilon: 0.005\n",
      "episode_num 1600, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.3, curr_epsilon: 0.005\n",
      "episode_num 1601, curr_reward: 2.0, best_reward: 10.0, running_avg_reward: -8.12, curr_epsilon: 0.005\n",
      "episode_num 1602, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.18, curr_epsilon: 0.005\n",
      "episode_num 1603, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.34, curr_epsilon: 0.005\n",
      "episode_num 1604, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.4, curr_epsilon: 0.005\n",
      "episode_num 1605, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.37, curr_epsilon: 0.005\n",
      "episode_num 1606, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.33, curr_epsilon: 0.005\n",
      "episode_num 1607, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.37, curr_epsilon: 0.005\n",
      "episode_num 1608, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.36, curr_epsilon: 0.005\n",
      "episode_num 1609, curr_reward: 4.0, best_reward: 10.0, running_avg_reward: -8.27, curr_epsilon: 0.005\n",
      "episode_num 1610, curr_reward: 7.0, best_reward: 10.0, running_avg_reward: -8.1, curr_epsilon: 0.005\n",
      "episode_num 1611, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.13, curr_epsilon: 0.005\n",
      "episode_num 1612, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.07, curr_epsilon: 0.005\n",
      "episode_num 1613, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.12, curr_epsilon: 0.005\n",
      "episode_num 1614, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.09, curr_epsilon: 0.005\n",
      "episode_num 1615, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.15, curr_epsilon: 0.005\n",
      "episode_num 1616, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.24, curr_epsilon: 0.005\n",
      "episode_num 1617, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.27, curr_epsilon: 0.005\n",
      "episode_num 1618, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.33, curr_epsilon: 0.005\n",
      "episode_num 1619, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.51, curr_epsilon: 0.005\n",
      "episode_num 1620, curr_reward: 3.0, best_reward: 10.0, running_avg_reward: -8.41, curr_epsilon: 0.005\n",
      "episode_num 1621, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.33, curr_epsilon: 0.005\n",
      "episode_num 1622, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.3, curr_epsilon: 0.005\n",
      "episode_num 1623, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.31, curr_epsilon: 0.005\n",
      "episode_num 1624, curr_reward: -17.0, best_reward: 10.0, running_avg_reward: -8.44, curr_epsilon: 0.005\n",
      "episode_num 1625, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.52, curr_epsilon: 0.005\n",
      "episode_num 1626, curr_reward: -18.0, best_reward: 10.0, running_avg_reward: -8.64, curr_epsilon: 0.005\n",
      "episode_num 1627, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.68, curr_epsilon: 0.005\n",
      "episode_num 1628, curr_reward: -19.0, best_reward: 10.0, running_avg_reward: -8.79, curr_epsilon: 0.005\n",
      "episode_num 1629, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.82, curr_epsilon: 0.005\n",
      "episode_num 1630, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.87, curr_epsilon: 0.005\n",
      "episode_num 1631, curr_reward: 1.0, best_reward: 10.0, running_avg_reward: -8.75, curr_epsilon: 0.005\n",
      "episode_num 1632, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.77, curr_epsilon: 0.005\n",
      "episode_num 1633, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.86, curr_epsilon: 0.005\n",
      "episode_num 1634, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.94, curr_epsilon: 0.005\n",
      "episode_num 1635, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.92, curr_epsilon: 0.005\n",
      "episode_num 1636, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.96, curr_epsilon: 0.005\n",
      "episode_num 1637, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.96, curr_epsilon: 0.005\n",
      "episode_num 1638, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -9.03, curr_epsilon: 0.005\n",
      "episode_num 1639, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.94, curr_epsilon: 0.005\n",
      "episode_num 1640, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.93, curr_epsilon: 0.005\n",
      "episode_num 1641, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.93, curr_epsilon: 0.005\n",
      "episode_num 1642, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.92, curr_epsilon: 0.005\n",
      "episode_num 1643, curr_reward: -17.0, best_reward: 10.0, running_avg_reward: -9.03, curr_epsilon: 0.005\n",
      "episode_num 1644, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -9.01, curr_epsilon: 0.005\n",
      "episode_num 1645, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -9.0, curr_epsilon: 0.005\n",
      "episode_num 1646, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -9.06, curr_epsilon: 0.005\n",
      "episode_num 1647, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.09, curr_epsilon: 0.005\n",
      "episode_num 1648, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -9.01, curr_epsilon: 0.005\n",
      "episode_num 1649, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.94, curr_epsilon: 0.005\n",
      "episode_num 1650, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.92, curr_epsilon: 0.005\n",
      "episode_num 1651, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.94, curr_epsilon: 0.005\n",
      "episode_num 1652, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.89, curr_epsilon: 0.005\n",
      "episode_num 1653, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.89, curr_epsilon: 0.005\n",
      "episode_num 1654, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.92, curr_epsilon: 0.005\n",
      "episode_num 1655, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.86, curr_epsilon: 0.005\n",
      "episode_num 1656, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.95, curr_epsilon: 0.005\n",
      "episode_num 1657, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.94, curr_epsilon: 0.005\n",
      "episode_num 1658, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.89, curr_epsilon: 0.005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 1659, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.88, curr_epsilon: 0.005\n",
      "episode_num 1660, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.87, curr_epsilon: 0.005\n",
      "episode_num 1661, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.85, curr_epsilon: 0.005\n",
      "episode_num 1662, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.89, curr_epsilon: 0.005\n",
      "episode_num 1663, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.91, curr_epsilon: 0.005\n",
      "episode_num 1664, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.98, curr_epsilon: 0.005\n",
      "episode_num 1665, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -9.09, curr_epsilon: 0.005\n",
      "episode_num 1666, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -9.14, curr_epsilon: 0.005\n",
      "episode_num 1667, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -9.09, curr_epsilon: 0.005\n",
      "episode_num 1668, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -9.04, curr_epsilon: 0.005\n",
      "episode_num 1669, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -9.17, curr_epsilon: 0.005\n",
      "episode_num 1670, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -9.34, curr_epsilon: 0.005\n",
      "episode_num 1671, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -9.44, curr_epsilon: 0.005\n",
      "episode_num 1672, curr_reward: 1.0, best_reward: 10.0, running_avg_reward: -9.36, curr_epsilon: 0.005\n",
      "episode_num 1673, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -9.37, curr_epsilon: 0.005\n",
      "episode_num 1674, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -9.35, curr_epsilon: 0.005\n",
      "episode_num 1675, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -9.31, curr_epsilon: 0.005\n",
      "episode_num 1676, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -9.24, curr_epsilon: 0.005\n",
      "episode_num 1677, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -9.19, curr_epsilon: 0.005\n",
      "episode_num 1678, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -9.25, curr_epsilon: 0.005\n",
      "episode_num 1679, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -9.16, curr_epsilon: 0.005\n",
      "episode_num 1680, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -9.1, curr_epsilon: 0.005\n",
      "episode_num 1681, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -9.02, curr_epsilon: 0.005\n",
      "episode_num 1682, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.01, curr_epsilon: 0.005\n",
      "episode_num 1683, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.95, curr_epsilon: 0.005\n",
      "episode_num 1684, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.88, curr_epsilon: 0.005\n",
      "episode_num 1685, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -8.95, curr_epsilon: 0.005\n",
      "episode_num 1686, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -9.01, curr_epsilon: 0.005\n",
      "episode_num 1687, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -9.01, curr_epsilon: 0.005\n",
      "episode_num 1688, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.96, curr_epsilon: 0.005\n",
      "episode_num 1689, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -8.9, curr_epsilon: 0.005\n",
      "episode_num 1690, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.89, curr_epsilon: 0.005\n",
      "episode_num 1691, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.84, curr_epsilon: 0.005\n",
      "episode_num 1692, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.82, curr_epsilon: 0.005\n",
      "episode_num 1693, curr_reward: -18.0, best_reward: 10.0, running_avg_reward: -8.93, curr_epsilon: 0.005\n",
      "episode_num 1694, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.87, curr_epsilon: 0.005\n",
      "episode_num 1695, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.95, curr_epsilon: 0.005\n",
      "episode_num 1696, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.85, curr_epsilon: 0.005\n",
      "episode_num 1697, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.8, curr_epsilon: 0.005\n",
      "episode_num 1698, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -8.85, curr_epsilon: 0.005\n",
      "episode_num 1699, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.82, curr_epsilon: 0.005\n",
      "episode_num 1700, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.73, curr_epsilon: 0.005\n",
      "episode_num 1701, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.84, curr_epsilon: 0.005\n",
      "episode_num 1702, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.8, curr_epsilon: 0.005\n",
      "episode_num 1703, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.8, curr_epsilon: 0.005\n",
      "episode_num 1704, curr_reward: 1.0, best_reward: 10.0, running_avg_reward: -8.64, curr_epsilon: 0.005\n",
      "episode_num 1705, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.67, curr_epsilon: 0.005\n",
      "episode_num 1706, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.71, curr_epsilon: 0.005\n",
      "episode_num 1707, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.69, curr_epsilon: 0.005\n",
      "episode_num 1708, curr_reward: 4.0, best_reward: 10.0, running_avg_reward: -8.58, curr_epsilon: 0.005\n",
      "episode_num 1709, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.77, curr_epsilon: 0.005\n",
      "episode_num 1710, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -8.85, curr_epsilon: 0.005\n",
      "episode_num 1711, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.83, curr_epsilon: 0.005\n",
      "episode_num 1712, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.84, curr_epsilon: 0.005\n",
      "episode_num 1713, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.84, curr_epsilon: 0.005\n",
      "episode_num 1714, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -8.78, curr_epsilon: 0.005\n",
      "episode_num 1715, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.8, curr_epsilon: 0.005\n",
      "episode_num 1716, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.77, curr_epsilon: 0.005\n",
      "episode_num 1717, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.8, curr_epsilon: 0.005\n",
      "episode_num 1718, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.79, curr_epsilon: 0.005\n",
      "episode_num 1719, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.83, curr_epsilon: 0.005\n",
      "episode_num 1720, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.95, curr_epsilon: 0.005\n",
      "episode_num 1721, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -9.04, curr_epsilon: 0.005\n",
      "episode_num 1722, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -9.01, curr_epsilon: 0.005\n",
      "episode_num 1723, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -9.06, curr_epsilon: 0.005\n",
      "episode_num 1724, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -9.0, curr_epsilon: 0.005\n",
      "episode_num 1725, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.93, curr_epsilon: 0.005\n",
      "episode_num 1726, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.89, curr_epsilon: 0.005\n",
      "episode_num 1727, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.91, curr_epsilon: 0.005\n",
      "episode_num 1728, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.78, curr_epsilon: 0.005\n",
      "episode_num 1729, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.77, curr_epsilon: 0.005\n",
      "episode_num 1730, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.72, curr_epsilon: 0.005\n",
      "episode_num 1731, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.8, curr_epsilon: 0.005\n",
      "episode_num 1732, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.72, curr_epsilon: 0.005\n",
      "episode_num 1733, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.72, curr_epsilon: 0.005\n",
      "episode_num 1734, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.67, curr_epsilon: 0.005\n",
      "episode_num 1735, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.63, curr_epsilon: 0.005\n",
      "episode_num 1736, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.67, curr_epsilon: 0.005\n",
      "episode_num 1737, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.72, curr_epsilon: 0.005\n",
      "episode_num 1738, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.75, curr_epsilon: 0.005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 1739, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -8.68, curr_epsilon: 0.005\n",
      "episode_num 1740, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.72, curr_epsilon: 0.005\n",
      "episode_num 1741, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.69, curr_epsilon: 0.005\n",
      "episode_num 1742, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.65, curr_epsilon: 0.005\n",
      "episode_num 1743, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.62, curr_epsilon: 0.005\n",
      "episode_num 1744, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -8.73, curr_epsilon: 0.005\n",
      "episode_num 1745, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.77, curr_epsilon: 0.005\n",
      "episode_num 1746, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -8.65, curr_epsilon: 0.005\n",
      "episode_num 1747, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -8.58, curr_epsilon: 0.005\n",
      "episode_num 1748, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.62, curr_epsilon: 0.005\n",
      "episode_num 1749, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -8.59, curr_epsilon: 0.005\n",
      "episode_num 1750, curr_reward: 7.0, best_reward: 10.0, running_avg_reward: -8.41, curr_epsilon: 0.005\n",
      "episode_num 1751, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.46, curr_epsilon: 0.005\n",
      "episode_num 1752, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.47, curr_epsilon: 0.005\n",
      "episode_num 1753, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -8.4, curr_epsilon: 0.005\n",
      "episode_num 1754, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.43, curr_epsilon: 0.005\n",
      "episode_num 1755, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.43, curr_epsilon: 0.005\n",
      "episode_num 1756, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.38, curr_epsilon: 0.005\n",
      "episode_num 1757, curr_reward: 3.0, best_reward: 10.0, running_avg_reward: -8.2, curr_epsilon: 0.005\n",
      "episode_num 1758, curr_reward: 2.0, best_reward: 10.0, running_avg_reward: -8.1, curr_epsilon: 0.005\n",
      "episode_num 1759, curr_reward: 4.0, best_reward: 10.0, running_avg_reward: -8.02, curr_epsilon: 0.005\n",
      "episode_num 1760, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -8.05, curr_epsilon: 0.005\n",
      "episode_num 1761, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.12, curr_epsilon: 0.005\n",
      "episode_num 1762, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.18, curr_epsilon: 0.005\n",
      "episode_num 1763, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.15, curr_epsilon: 0.005\n",
      "episode_num 1764, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.21, curr_epsilon: 0.005\n",
      "episode_num 1765, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.18, curr_epsilon: 0.005\n",
      "episode_num 1766, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.15, curr_epsilon: 0.005\n",
      "episode_num 1767, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.15, curr_epsilon: 0.005\n",
      "episode_num 1768, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -8.09, curr_epsilon: 0.005\n",
      "episode_num 1769, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.08, curr_epsilon: 0.005\n",
      "episode_num 1770, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.01, curr_epsilon: 0.005\n",
      "episode_num 1771, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.99, curr_epsilon: 0.005\n",
      "episode_num 1772, curr_reward: 4.0, best_reward: 10.0, running_avg_reward: -7.96, curr_epsilon: 0.005\n",
      "episode_num 1773, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.95, curr_epsilon: 0.005\n",
      "episode_num 1774, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.9, curr_epsilon: 0.005\n",
      "episode_num 1775, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.89, curr_epsilon: 0.005\n",
      "episode_num 1776, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.94, curr_epsilon: 0.005\n",
      "episode_num 1777, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -8.07, curr_epsilon: 0.005\n",
      "episode_num 1778, curr_reward: 3.0, best_reward: 10.0, running_avg_reward: -7.92, curr_epsilon: 0.005\n",
      "episode_num 1779, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -7.92, curr_epsilon: 0.005\n",
      "episode_num 1780, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -7.88, curr_epsilon: 0.005\n",
      "episode_num 1781, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -7.97, curr_epsilon: 0.005\n",
      "episode_num 1782, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.92, curr_epsilon: 0.005\n",
      "episode_num 1783, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -7.85, curr_epsilon: 0.005\n",
      "episode_num 1784, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.84, curr_epsilon: 0.005\n",
      "episode_num 1785, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.77, curr_epsilon: 0.005\n",
      "episode_num 1786, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.78, curr_epsilon: 0.005\n",
      "episode_num 1787, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.84, curr_epsilon: 0.005\n",
      "episode_num 1788, curr_reward: 1.0, best_reward: 10.0, running_avg_reward: -7.77, curr_epsilon: 0.005\n",
      "episode_num 1789, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.83, curr_epsilon: 0.005\n",
      "episode_num 1790, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.79, curr_epsilon: 0.005\n",
      "episode_num 1791, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -7.87, curr_epsilon: 0.005\n",
      "episode_num 1792, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.86, curr_epsilon: 0.005\n",
      "episode_num 1793, curr_reward: 5.0, best_reward: 10.0, running_avg_reward: -7.63, curr_epsilon: 0.005\n",
      "episode_num 1794, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -7.62, curr_epsilon: 0.005\n",
      "episode_num 1795, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -7.59, curr_epsilon: 0.005\n",
      "episode_num 1796, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.62, curr_epsilon: 0.005\n",
      "episode_num 1797, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.68, curr_epsilon: 0.005\n",
      "episode_num 1798, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -7.58, curr_epsilon: 0.005\n",
      "episode_num 1799, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -7.55, curr_epsilon: 0.005\n",
      "episode_num 1800, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -7.61, curr_epsilon: 0.005\n",
      "episode_num 1801, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -7.65, curr_epsilon: 0.005\n",
      "episode_num 1802, curr_reward: 2.0, best_reward: 10.0, running_avg_reward: -7.56, curr_epsilon: 0.005\n",
      "episode_num 1803, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -7.55, curr_epsilon: 0.005\n",
      "episode_num 1804, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.6, curr_epsilon: 0.005\n",
      "episode_num 1805, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.51, curr_epsilon: 0.005\n",
      "episode_num 1806, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.42, curr_epsilon: 0.005\n",
      "episode_num 1807, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -7.51, curr_epsilon: 0.005\n",
      "episode_num 1808, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -7.7, curr_epsilon: 0.005\n",
      "episode_num 1809, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.62, curr_epsilon: 0.005\n",
      "episode_num 1810, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -7.71, curr_epsilon: 0.005\n",
      "episode_num 1811, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -7.76, curr_epsilon: 0.005\n",
      "episode_num 1812, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -7.83, curr_epsilon: 0.005\n",
      "episode_num 1813, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -7.86, curr_epsilon: 0.005\n",
      "episode_num 1814, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -7.9, curr_epsilon: 0.005\n",
      "episode_num 1815, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -7.89, curr_epsilon: 0.005\n",
      "episode_num 1816, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -7.81, curr_epsilon: 0.005\n",
      "episode_num 1817, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.77, curr_epsilon: 0.005\n",
      "episode_num 1818, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -7.69, curr_epsilon: 0.005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 1819, curr_reward: 4.0, best_reward: 10.0, running_avg_reward: -7.5, curr_epsilon: 0.005\n",
      "episode_num 1820, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.45, curr_epsilon: 0.005\n",
      "episode_num 1821, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -7.38, curr_epsilon: 0.005\n",
      "episode_num 1822, curr_reward: 2.0, best_reward: 10.0, running_avg_reward: -7.34, curr_epsilon: 0.005\n",
      "episode_num 1823, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.32, curr_epsilon: 0.005\n",
      "episode_num 1824, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.3, curr_epsilon: 0.005\n",
      "episode_num 1825, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -7.37, curr_epsilon: 0.005\n",
      "episode_num 1826, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.31, curr_epsilon: 0.005\n",
      "episode_num 1827, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -7.33, curr_epsilon: 0.005\n",
      "episode_num 1828, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.39, curr_epsilon: 0.005\n",
      "episode_num 1829, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.37, curr_epsilon: 0.005\n",
      "episode_num 1830, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -7.38, curr_epsilon: 0.005\n",
      "episode_num 1831, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -7.46, curr_epsilon: 0.005\n",
      "episode_num 1832, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -7.47, curr_epsilon: 0.005\n",
      "episode_num 1833, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -7.47, curr_epsilon: 0.005\n",
      "episode_num 1834, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -7.48, curr_epsilon: 0.005\n",
      "episode_num 1835, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.46, curr_epsilon: 0.005\n",
      "episode_num 1836, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -7.45, curr_epsilon: 0.005\n",
      "episode_num 1837, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -7.37, curr_epsilon: 0.005\n",
      "episode_num 1838, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -7.37, curr_epsilon: 0.005\n",
      "episode_num 1839, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.43, curr_epsilon: 0.005\n",
      "episode_num 1840, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -7.36, curr_epsilon: 0.005\n",
      "episode_num 1841, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -7.35, curr_epsilon: 0.005\n",
      "episode_num 1842, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -7.4, curr_epsilon: 0.005\n",
      "episode_num 1843, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -7.28, curr_epsilon: 0.005\n",
      "episode_num 1844, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -7.18, curr_epsilon: 0.005\n",
      "episode_num 1845, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -7.05, curr_epsilon: 0.005\n",
      "episode_num 1846, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -7.19, curr_epsilon: 0.005\n",
      "episode_num 1847, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.25, curr_epsilon: 0.005\n",
      "episode_num 1848, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -7.24, curr_epsilon: 0.005\n",
      "episode_num 1849, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.29, curr_epsilon: 0.005\n",
      "episode_num 1850, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.45, curr_epsilon: 0.005\n",
      "episode_num 1851, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -7.35, curr_epsilon: 0.005\n",
      "episode_num 1852, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.33, curr_epsilon: 0.005\n",
      "episode_num 1853, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -7.36, curr_epsilon: 0.005\n",
      "episode_num 1854, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -7.33, curr_epsilon: 0.005\n",
      "episode_num 1855, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.39, curr_epsilon: 0.005\n",
      "episode_num 1856, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.43, curr_epsilon: 0.005\n",
      "episode_num 1857, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -7.47, curr_epsilon: 0.005\n",
      "episode_num 1858, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -7.6, curr_epsilon: 0.005\n",
      "episode_num 1859, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.72, curr_epsilon: 0.005\n",
      "episode_num 1860, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -7.57, curr_epsilon: 0.005\n",
      "episode_num 1861, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.51, curr_epsilon: 0.005\n",
      "episode_num 1862, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.42, curr_epsilon: 0.005\n",
      "episode_num 1863, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.39, curr_epsilon: 0.005\n",
      "episode_num 1864, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -7.41, curr_epsilon: 0.005\n",
      "episode_num 1865, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -7.32, curr_epsilon: 0.005\n",
      "episode_num 1866, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -7.35, curr_epsilon: 0.005\n",
      "episode_num 1867, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.37, curr_epsilon: 0.005\n",
      "episode_num 1868, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.4, curr_epsilon: 0.005\n",
      "episode_num 1869, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.32, curr_epsilon: 0.005\n",
      "episode_num 1870, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.37, curr_epsilon: 0.005\n",
      "episode_num 1871, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -7.38, curr_epsilon: 0.005\n",
      "episode_num 1872, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.49, curr_epsilon: 0.005\n",
      "episode_num 1873, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.47, curr_epsilon: 0.005\n",
      "episode_num 1874, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.47, curr_epsilon: 0.005\n",
      "episode_num 1875, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.5, curr_epsilon: 0.005\n",
      "episode_num 1876, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -7.58, curr_epsilon: 0.005\n",
      "episode_num 1877, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -7.44, curr_epsilon: 0.005\n",
      "episode_num 1878, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.54, curr_epsilon: 0.005\n",
      "episode_num 1879, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -7.55, curr_epsilon: 0.005\n",
      "episode_num 1880, curr_reward: 5.0, best_reward: 10.0, running_avg_reward: -7.49, curr_epsilon: 0.005\n",
      "episode_num 1881, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -7.39, curr_epsilon: 0.005\n",
      "episode_num 1882, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.47, curr_epsilon: 0.005\n",
      "episode_num 1883, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.53, curr_epsilon: 0.005\n",
      "episode_num 1884, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -7.6, curr_epsilon: 0.005\n",
      "episode_num 1885, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.63, curr_epsilon: 0.005\n",
      "episode_num 1886, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -7.65, curr_epsilon: 0.005\n",
      "episode_num 1887, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -7.66, curr_epsilon: 0.005\n",
      "episode_num 1888, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -7.68, curr_epsilon: 0.005\n",
      "episode_num 1889, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.67, curr_epsilon: 0.005\n",
      "episode_num 1890, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.69, curr_epsilon: 0.005\n",
      "episode_num 1891, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -7.71, curr_epsilon: 0.005\n",
      "episode_num 1892, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -7.67, curr_epsilon: 0.005\n",
      "episode_num 1893, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.81, curr_epsilon: 0.005\n",
      "episode_num 1894, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -7.86, curr_epsilon: 0.005\n",
      "episode_num 1895, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -7.87, curr_epsilon: 0.005\n",
      "episode_num 1896, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -7.84, curr_epsilon: 0.005\n",
      "episode_num 1897, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.76, curr_epsilon: 0.005\n",
      "episode_num 1898, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.77, curr_epsilon: 0.005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 1899, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -7.72, curr_epsilon: 0.005\n",
      "episode_num 1900, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.69, curr_epsilon: 0.005\n",
      "episode_num 1901, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -7.58, curr_epsilon: 0.005\n",
      "episode_num 1902, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -7.62, curr_epsilon: 0.005\n",
      "episode_num 1903, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.57, curr_epsilon: 0.005\n",
      "episode_num 1904, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.6, curr_epsilon: 0.005\n",
      "episode_num 1905, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.65, curr_epsilon: 0.005\n",
      "episode_num 1906, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -7.64, curr_epsilon: 0.005\n",
      "episode_num 1907, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.57, curr_epsilon: 0.005\n",
      "episode_num 1908, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.51, curr_epsilon: 0.005\n",
      "episode_num 1909, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -7.59, curr_epsilon: 0.005\n",
      "episode_num 1910, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -7.51, curr_epsilon: 0.005\n",
      "episode_num 1911, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.47, curr_epsilon: 0.005\n",
      "episode_num 1912, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -7.43, curr_epsilon: 0.005\n",
      "episode_num 1913, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -7.43, curr_epsilon: 0.005\n",
      "episode_num 1914, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -7.48, curr_epsilon: 0.005\n",
      "episode_num 1915, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.47, curr_epsilon: 0.005\n",
      "episode_num 1916, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.51, curr_epsilon: 0.005\n",
      "episode_num 1917, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -7.56, curr_epsilon: 0.005\n",
      "episode_num 1918, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.58, curr_epsilon: 0.005\n",
      "episode_num 1919, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.71, curr_epsilon: 0.005\n",
      "episode_num 1920, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -7.82, curr_epsilon: 0.005\n",
      "episode_num 1921, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.8, curr_epsilon: 0.005\n",
      "episode_num 1922, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -7.96, curr_epsilon: 0.005\n",
      "episode_num 1923, curr_reward: -19.0, best_reward: 10.0, running_avg_reward: -8.06, curr_epsilon: 0.005\n",
      "episode_num 1924, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.03, curr_epsilon: 0.005\n",
      "episode_num 1925, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.01, curr_epsilon: 0.005\n",
      "episode_num 1926, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -7.95, curr_epsilon: 0.005\n",
      "episode_num 1927, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -7.98, curr_epsilon: 0.005\n",
      "episode_num 1928, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.01, curr_epsilon: 0.005\n",
      "episode_num 1929, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.05, curr_epsilon: 0.005\n",
      "episode_num 1930, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.06, curr_epsilon: 0.005\n",
      "episode_num 1931, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.0, curr_epsilon: 0.005\n",
      "episode_num 1932, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.03, curr_epsilon: 0.005\n",
      "episode_num 1933, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -7.97, curr_epsilon: 0.005\n",
      "episode_num 1934, curr_reward: 10.0, best_reward: 10.0, running_avg_reward: -7.82, curr_epsilon: 0.005\n",
      "episode_num 1935, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -7.87, curr_epsilon: 0.005\n",
      "episode_num 1936, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -7.88, curr_epsilon: 0.005\n",
      "episode_num 1937, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -7.84, curr_epsilon: 0.005\n",
      "episode_num 1938, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.77, curr_epsilon: 0.005\n",
      "episode_num 1939, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -7.82, curr_epsilon: 0.005\n",
      "episode_num 1940, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -7.87, curr_epsilon: 0.005\n",
      "episode_num 1941, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -7.92, curr_epsilon: 0.005\n",
      "episode_num 1942, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -7.91, curr_epsilon: 0.005\n",
      "episode_num 1943, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -7.97, curr_epsilon: 0.005\n",
      "episode_num 1944, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.04, curr_epsilon: 0.005\n",
      "episode_num 1945, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.08, curr_epsilon: 0.005\n",
      "episode_num 1946, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -8.02, curr_epsilon: 0.005\n",
      "episode_num 1947, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.02, curr_epsilon: 0.005\n",
      "episode_num 1948, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -7.99, curr_epsilon: 0.005\n",
      "episode_num 1949, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -7.99, curr_epsilon: 0.005\n",
      "episode_num 1950, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.04, curr_epsilon: 0.005\n",
      "episode_num 1951, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.12, curr_epsilon: 0.005\n",
      "episode_num 1952, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.12, curr_epsilon: 0.005\n",
      "episode_num 1953, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -8.08, curr_epsilon: 0.005\n",
      "episode_num 1954, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.11, curr_epsilon: 0.005\n",
      "episode_num 1955, curr_reward: 3.0, best_reward: 10.0, running_avg_reward: -7.96, curr_epsilon: 0.005\n",
      "episode_num 1956, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -7.98, curr_epsilon: 0.005\n",
      "episode_num 1957, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.05, curr_epsilon: 0.005\n",
      "episode_num 1958, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.08, curr_epsilon: 0.005\n",
      "episode_num 1959, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.07, curr_epsilon: 0.005\n",
      "episode_num 1960, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.16, curr_epsilon: 0.005\n",
      "episode_num 1961, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.23, curr_epsilon: 0.005\n",
      "episode_num 1962, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.24, curr_epsilon: 0.005\n",
      "episode_num 1963, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.22, curr_epsilon: 0.005\n",
      "episode_num 1964, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.16, curr_epsilon: 0.005\n",
      "episode_num 1965, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.23, curr_epsilon: 0.005\n",
      "episode_num 1966, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.24, curr_epsilon: 0.005\n",
      "episode_num 1967, curr_reward: -18.0, best_reward: 10.0, running_avg_reward: -8.33, curr_epsilon: 0.005\n",
      "episode_num 1968, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.43, curr_epsilon: 0.005\n",
      "episode_num 1969, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -8.37, curr_epsilon: 0.005\n",
      "episode_num 1970, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.34, curr_epsilon: 0.005\n",
      "episode_num 1971, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.36, curr_epsilon: 0.005\n",
      "episode_num 1972, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.36, curr_epsilon: 0.005\n",
      "episode_num 1973, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.4, curr_epsilon: 0.005\n",
      "episode_num 1974, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.43, curr_epsilon: 0.005\n",
      "episode_num 1975, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -8.38, curr_epsilon: 0.005\n",
      "episode_num 1976, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.33, curr_epsilon: 0.005\n",
      "episode_num 1977, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -8.47, curr_epsilon: 0.005\n",
      "episode_num 1978, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.47, curr_epsilon: 0.005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 1979, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -8.55, curr_epsilon: 0.005\n",
      "episode_num 1980, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.64, curr_epsilon: 0.005\n",
      "episode_num 1981, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -8.71, curr_epsilon: 0.005\n",
      "episode_num 1982, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -8.75, curr_epsilon: 0.005\n",
      "episode_num 1983, curr_reward: -6.0, best_reward: 10.0, running_avg_reward: -8.73, curr_epsilon: 0.005\n",
      "episode_num 1984, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.7, curr_epsilon: 0.005\n",
      "episode_num 1985, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.66, curr_epsilon: 0.005\n",
      "episode_num 1986, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -8.67, curr_epsilon: 0.005\n",
      "episode_num 1987, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -8.62, curr_epsilon: 0.005\n",
      "episode_num 1988, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.74, curr_epsilon: 0.005\n",
      "episode_num 1989, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -8.81, curr_epsilon: 0.005\n",
      "episode_num 1990, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.85, curr_epsilon: 0.005\n",
      "episode_num 1991, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.82, curr_epsilon: 0.005\n",
      "episode_num 1992, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -8.81, curr_epsilon: 0.005\n",
      "episode_num 1993, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -8.84, curr_epsilon: 0.005\n",
      "episode_num 1994, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -8.86, curr_epsilon: 0.005\n",
      "episode_num 1995, curr_reward: -17.0, best_reward: 10.0, running_avg_reward: -8.92, curr_epsilon: 0.005\n",
      "episode_num 1996, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -8.92, curr_epsilon: 0.005\n",
      "episode_num 1997, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -8.99, curr_epsilon: 0.005\n",
      "episode_num 1998, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -9.06, curr_epsilon: 0.005\n",
      "episode_num 1999, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -9.16, curr_epsilon: 0.005\n",
      "episode_num 2000, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -9.21, curr_epsilon: 0.005\n",
      "episode_num 2001, curr_reward: -14.0, best_reward: 10.0, running_avg_reward: -9.33, curr_epsilon: 0.005\n",
      "episode_num 2002, curr_reward: -16.0, best_reward: 10.0, running_avg_reward: -9.47, curr_epsilon: 0.005\n",
      "episode_num 2003, curr_reward: -12.0, best_reward: 10.0, running_avg_reward: -9.51, curr_epsilon: 0.005\n",
      "episode_num 2004, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.53, curr_epsilon: 0.005\n",
      "episode_num 2005, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -9.55, curr_epsilon: 0.005\n",
      "episode_num 2006, curr_reward: -15.0, best_reward: 10.0, running_avg_reward: -9.67, curr_epsilon: 0.005\n",
      "episode_num 2007, curr_reward: -18.0, best_reward: 10.0, running_avg_reward: -9.77, curr_epsilon: 0.005\n",
      "episode_num 2008, curr_reward: -1.0, best_reward: 10.0, running_avg_reward: -9.69, curr_epsilon: 0.005\n",
      "episode_num 2009, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -9.57, curr_epsilon: 0.005\n",
      "episode_num 2010, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -9.66, curr_epsilon: 0.005\n",
      "episode_num 2011, curr_reward: -11.0, best_reward: 10.0, running_avg_reward: -9.7, curr_epsilon: 0.005\n",
      "episode_num 2012, curr_reward: -2.0, best_reward: 10.0, running_avg_reward: -9.62, curr_epsilon: 0.005\n",
      "episode_num 2013, curr_reward: -4.0, best_reward: 10.0, running_avg_reward: -9.52, curr_epsilon: 0.005\n",
      "episode_num 2014, curr_reward: -18.0, best_reward: 10.0, running_avg_reward: -9.6, curr_epsilon: 0.005\n",
      "episode_num 2015, curr_reward: -13.0, best_reward: 10.0, running_avg_reward: -9.64, curr_epsilon: 0.005\n",
      "episode_num 2016, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -9.6, curr_epsilon: 0.005\n",
      "episode_num 2017, curr_reward: -17.0, best_reward: 10.0, running_avg_reward: -9.64, curr_epsilon: 0.005\n",
      "episode_num 2018, curr_reward: -10.0, best_reward: 10.0, running_avg_reward: -9.67, curr_epsilon: 0.005\n",
      "episode_num 2019, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.67, curr_epsilon: 0.005\n",
      "episode_num 2020, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.61, curr_epsilon: 0.005\n",
      "episode_num 2021, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -9.65, curr_epsilon: 0.005\n",
      "episode_num 2022, curr_reward: -3.0, best_reward: 10.0, running_avg_reward: -9.54, curr_epsilon: 0.005\n",
      "episode_num 2023, curr_reward: -9.0, best_reward: 10.0, running_avg_reward: -9.44, curr_epsilon: 0.005\n",
      "episode_num 2024, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -9.43, curr_epsilon: 0.005\n",
      "episode_num 2025, curr_reward: -17.0, best_reward: 10.0, running_avg_reward: -9.48, curr_epsilon: 0.005\n",
      "episode_num 2026, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -9.51, curr_epsilon: 0.005\n",
      "episode_num 2027, curr_reward: -7.0, best_reward: 10.0, running_avg_reward: -9.42, curr_epsilon: 0.005\n",
      "episode_num 2028, curr_reward: -5.0, best_reward: 10.0, running_avg_reward: -9.32, curr_epsilon: 0.005\n",
      "episode_num 2029, curr_reward: -8.0, best_reward: 10.0, running_avg_reward: -9.28, curr_epsilon: 0.005\n",
      "episode_num 2030, curr_reward: 11.0, best_reward: 11.0, running_avg_reward: -9.1, curr_epsilon: 0.005\n",
      "episode_num 2031, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -9.1, curr_epsilon: 0.005\n",
      "episode_num 2032, curr_reward: -4.0, best_reward: 11.0, running_avg_reward: -9.05, curr_epsilon: 0.005\n",
      "episode_num 2033, curr_reward: -13.0, best_reward: 11.0, running_avg_reward: -9.14, curr_epsilon: 0.005\n",
      "episode_num 2034, curr_reward: -13.0, best_reward: 11.0, running_avg_reward: -9.37, curr_epsilon: 0.005\n",
      "episode_num 2035, curr_reward: -1.0, best_reward: 11.0, running_avg_reward: -9.29, curr_epsilon: 0.005\n",
      "episode_num 2036, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -9.28, curr_epsilon: 0.005\n",
      "episode_num 2037, curr_reward: -13.0, best_reward: 11.0, running_avg_reward: -9.4, curr_epsilon: 0.005\n",
      "episode_num 2038, curr_reward: -6.0, best_reward: 11.0, running_avg_reward: -9.38, curr_epsilon: 0.005\n",
      "episode_num 2039, curr_reward: -13.0, best_reward: 11.0, running_avg_reward: -9.38, curr_epsilon: 0.005\n",
      "episode_num 2040, curr_reward: -14.0, best_reward: 11.0, running_avg_reward: -9.42, curr_epsilon: 0.005\n",
      "episode_num 2041, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -9.43, curr_epsilon: 0.005\n",
      "episode_num 2042, curr_reward: -13.0, best_reward: 11.0, running_avg_reward: -9.44, curr_epsilon: 0.005\n",
      "episode_num 2043, curr_reward: -5.0, best_reward: 11.0, running_avg_reward: -9.41, curr_epsilon: 0.005\n",
      "episode_num 2044, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -9.39, curr_epsilon: 0.005\n",
      "episode_num 2045, curr_reward: -14.0, best_reward: 11.0, running_avg_reward: -9.47, curr_epsilon: 0.005\n",
      "episode_num 2046, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -9.49, curr_epsilon: 0.005\n",
      "episode_num 2047, curr_reward: 1.0, best_reward: 11.0, running_avg_reward: -9.4, curr_epsilon: 0.005\n",
      "episode_num 2048, curr_reward: -7.0, best_reward: 11.0, running_avg_reward: -9.44, curr_epsilon: 0.005\n",
      "episode_num 2049, curr_reward: -13.0, best_reward: 11.0, running_avg_reward: -9.5, curr_epsilon: 0.005\n",
      "episode_num 2050, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -9.45, curr_epsilon: 0.005\n",
      "episode_num 2051, curr_reward: -8.0, best_reward: 11.0, running_avg_reward: -9.43, curr_epsilon: 0.005\n",
      "episode_num 2052, curr_reward: -13.0, best_reward: 11.0, running_avg_reward: -9.48, curr_epsilon: 0.005\n",
      "episode_num 2053, curr_reward: -13.0, best_reward: 11.0, running_avg_reward: -9.59, curr_epsilon: 0.005\n",
      "episode_num 2054, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -9.54, curr_epsilon: 0.005\n",
      "episode_num 2055, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -9.66, curr_epsilon: 0.005\n",
      "episode_num 2056, curr_reward: -1.0, best_reward: 11.0, running_avg_reward: -9.56, curr_epsilon: 0.005\n",
      "episode_num 2057, curr_reward: -6.0, best_reward: 11.0, running_avg_reward: -9.54, curr_epsilon: 0.005\n",
      "episode_num 2058, curr_reward: -3.0, best_reward: 11.0, running_avg_reward: -9.43, curr_epsilon: 0.005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 2059, curr_reward: -12.0, best_reward: 11.0, running_avg_reward: -9.48, curr_epsilon: 0.005\n",
      "episode_num 2060, curr_reward: -4.0, best_reward: 11.0, running_avg_reward: -9.42, curr_epsilon: 0.005\n",
      "episode_num 2061, curr_reward: -4.0, best_reward: 11.0, running_avg_reward: -9.31, curr_epsilon: 0.005\n",
      "episode_num 2062, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -9.37, curr_epsilon: 0.005\n",
      "episode_num 2063, curr_reward: -1.0, best_reward: 11.0, running_avg_reward: -9.33, curr_epsilon: 0.005\n",
      "episode_num 2064, curr_reward: -3.0, best_reward: 11.0, running_avg_reward: -9.26, curr_epsilon: 0.005\n",
      "episode_num 2065, curr_reward: -6.0, best_reward: 11.0, running_avg_reward: -9.24, curr_epsilon: 0.005\n",
      "episode_num 2066, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -9.21, curr_epsilon: 0.005\n",
      "episode_num 2067, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -9.12, curr_epsilon: 0.005\n",
      "episode_num 2068, curr_reward: -7.0, best_reward: 11.0, running_avg_reward: -9.05, curr_epsilon: 0.005\n",
      "episode_num 2069, curr_reward: -7.0, best_reward: 11.0, running_avg_reward: -9.11, curr_epsilon: 0.005\n",
      "episode_num 2070, curr_reward: -14.0, best_reward: 11.0, running_avg_reward: -9.19, curr_epsilon: 0.005\n",
      "episode_num 2071, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -9.14, curr_epsilon: 0.005\n",
      "episode_num 2072, curr_reward: -12.0, best_reward: 11.0, running_avg_reward: -9.19, curr_epsilon: 0.005\n",
      "episode_num 2073, curr_reward: -5.0, best_reward: 11.0, running_avg_reward: -9.13, curr_epsilon: 0.005\n",
      "episode_num 2074, curr_reward: -4.0, best_reward: 11.0, running_avg_reward: -9.06, curr_epsilon: 0.005\n",
      "episode_num 2075, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -9.15, curr_epsilon: 0.005\n",
      "episode_num 2076, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -9.13, curr_epsilon: 0.005\n",
      "episode_num 2077, curr_reward: -5.0, best_reward: 11.0, running_avg_reward: -9.02, curr_epsilon: 0.005\n",
      "episode_num 2078, curr_reward: -14.0, best_reward: 11.0, running_avg_reward: -9.09, curr_epsilon: 0.005\n",
      "episode_num 2079, curr_reward: -13.0, best_reward: 11.0, running_avg_reward: -9.08, curr_epsilon: 0.005\n",
      "episode_num 2080, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -9.14, curr_epsilon: 0.005\n",
      "episode_num 2081, curr_reward: -6.0, best_reward: 11.0, running_avg_reward: -9.1, curr_epsilon: 0.005\n",
      "episode_num 2082, curr_reward: -4.0, best_reward: 11.0, running_avg_reward: -8.98, curr_epsilon: 0.005\n",
      "episode_num 2083, curr_reward: -7.0, best_reward: 11.0, running_avg_reward: -8.99, curr_epsilon: 0.005\n",
      "episode_num 2084, curr_reward: -6.0, best_reward: 11.0, running_avg_reward: -8.97, curr_epsilon: 0.005\n",
      "episode_num 2085, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -8.98, curr_epsilon: 0.005\n",
      "episode_num 2086, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -9.0, curr_epsilon: 0.005\n",
      "episode_num 2087, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -9.01, curr_epsilon: 0.005\n",
      "episode_num 2088, curr_reward: -12.0, best_reward: 11.0, running_avg_reward: -9.0, curr_epsilon: 0.005\n",
      "episode_num 2089, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -8.96, curr_epsilon: 0.005\n",
      "episode_num 2090, curr_reward: -8.0, best_reward: 11.0, running_avg_reward: -8.91, curr_epsilon: 0.005\n",
      "episode_num 2091, curr_reward: -5.0, best_reward: 11.0, running_avg_reward: -8.84, curr_epsilon: 0.005\n",
      "episode_num 2092, curr_reward: -1.0, best_reward: 11.0, running_avg_reward: -8.81, curr_epsilon: 0.005\n",
      "episode_num 2093, curr_reward: -16.0, best_reward: 11.0, running_avg_reward: -8.85, curr_epsilon: 0.005\n",
      "episode_num 2094, curr_reward: -6.0, best_reward: 11.0, running_avg_reward: -8.78, curr_epsilon: 0.005\n",
      "episode_num 2095, curr_reward: -16.0, best_reward: 11.0, running_avg_reward: -8.77, curr_epsilon: 0.005\n",
      "episode_num 2096, curr_reward: -7.0, best_reward: 11.0, running_avg_reward: -8.79, curr_epsilon: 0.005\n",
      "episode_num 2097, curr_reward: -12.0, best_reward: 11.0, running_avg_reward: -8.8, curr_epsilon: 0.005\n",
      "episode_num 2098, curr_reward: -1.0, best_reward: 11.0, running_avg_reward: -8.67, curr_epsilon: 0.005\n",
      "episode_num 2099, curr_reward: 3.0, best_reward: 11.0, running_avg_reward: -8.49, curr_epsilon: 0.005\n",
      "episode_num 2100, curr_reward: -7.0, best_reward: 11.0, running_avg_reward: -8.44, curr_epsilon: 0.005\n",
      "episode_num 2101, curr_reward: -2.0, best_reward: 11.0, running_avg_reward: -8.32, curr_epsilon: 0.005\n",
      "episode_num 2102, curr_reward: -8.0, best_reward: 11.0, running_avg_reward: -8.24, curr_epsilon: 0.005\n",
      "episode_num 2103, curr_reward: -8.0, best_reward: 11.0, running_avg_reward: -8.2, curr_epsilon: 0.005\n",
      "episode_num 2104, curr_reward: 3.0, best_reward: 11.0, running_avg_reward: -8.08, curr_epsilon: 0.005\n",
      "episode_num 2105, curr_reward: -14.0, best_reward: 11.0, running_avg_reward: -8.11, curr_epsilon: 0.005\n",
      "episode_num 2106, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -8.07, curr_epsilon: 0.005\n",
      "episode_num 2107, curr_reward: -6.0, best_reward: 11.0, running_avg_reward: -7.95, curr_epsilon: 0.005\n",
      "episode_num 2108, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -8.05, curr_epsilon: 0.005\n",
      "episode_num 2109, curr_reward: 10.0, best_reward: 11.0, running_avg_reward: -7.92, curr_epsilon: 0.005\n",
      "episode_num 2110, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.91, curr_epsilon: 0.005\n",
      "episode_num 2111, curr_reward: -5.0, best_reward: 11.0, running_avg_reward: -7.85, curr_epsilon: 0.005\n",
      "episode_num 2112, curr_reward: -7.0, best_reward: 11.0, running_avg_reward: -7.9, curr_epsilon: 0.005\n",
      "episode_num 2113, curr_reward: -6.0, best_reward: 11.0, running_avg_reward: -7.92, curr_epsilon: 0.005\n",
      "episode_num 2114, curr_reward: -14.0, best_reward: 11.0, running_avg_reward: -7.88, curr_epsilon: 0.005\n",
      "episode_num 2115, curr_reward: -2.0, best_reward: 11.0, running_avg_reward: -7.77, curr_epsilon: 0.005\n",
      "episode_num 2116, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -7.85, curr_epsilon: 0.005\n",
      "episode_num 2117, curr_reward: -5.0, best_reward: 11.0, running_avg_reward: -7.73, curr_epsilon: 0.005\n",
      "episode_num 2118, curr_reward: -7.0, best_reward: 11.0, running_avg_reward: -7.7, curr_epsilon: 0.005\n",
      "episode_num 2119, curr_reward: -4.0, best_reward: 11.0, running_avg_reward: -7.65, curr_epsilon: 0.005\n",
      "episode_num 2120, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.66, curr_epsilon: 0.005\n",
      "episode_num 2121, curr_reward: -15.0, best_reward: 11.0, running_avg_reward: -7.73, curr_epsilon: 0.005\n",
      "episode_num 2122, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -7.81, curr_epsilon: 0.005\n",
      "episode_num 2123, curr_reward: -6.0, best_reward: 11.0, running_avg_reward: -7.78, curr_epsilon: 0.005\n",
      "episode_num 2124, curr_reward: -17.0, best_reward: 11.0, running_avg_reward: -7.9, curr_epsilon: 0.005\n",
      "episode_num 2125, curr_reward: -15.0, best_reward: 11.0, running_avg_reward: -7.88, curr_epsilon: 0.005\n",
      "episode_num 2126, curr_reward: 6.0, best_reward: 11.0, running_avg_reward: -7.77, curr_epsilon: 0.005\n",
      "episode_num 2127, curr_reward: 2.0, best_reward: 11.0, running_avg_reward: -7.68, curr_epsilon: 0.005\n",
      "episode_num 2128, curr_reward: 1.0, best_reward: 11.0, running_avg_reward: -7.62, curr_epsilon: 0.005\n",
      "episode_num 2129, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -7.63, curr_epsilon: 0.005\n",
      "episode_num 2130, curr_reward: -8.0, best_reward: 11.0, running_avg_reward: -7.82, curr_epsilon: 0.005\n",
      "episode_num 2131, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -7.84, curr_epsilon: 0.005\n",
      "episode_num 2132, curr_reward: -3.0, best_reward: 11.0, running_avg_reward: -7.83, curr_epsilon: 0.005\n",
      "episode_num 2133, curr_reward: -17.0, best_reward: 11.0, running_avg_reward: -7.87, curr_epsilon: 0.005\n",
      "episode_num 2134, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.84, curr_epsilon: 0.005\n",
      "episode_num 2135, curr_reward: -8.0, best_reward: 11.0, running_avg_reward: -7.91, curr_epsilon: 0.005\n",
      "episode_num 2136, curr_reward: -15.0, best_reward: 11.0, running_avg_reward: -7.96, curr_epsilon: 0.005\n",
      "episode_num 2137, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -7.92, curr_epsilon: 0.005\n",
      "episode_num 2138, curr_reward: -1.0, best_reward: 11.0, running_avg_reward: -7.87, curr_epsilon: 0.005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode_num 2139, curr_reward: -5.0, best_reward: 11.0, running_avg_reward: -7.79, curr_epsilon: 0.005\n",
      "episode_num 2140, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.75, curr_epsilon: 0.005\n",
      "episode_num 2141, curr_reward: 10.0, best_reward: 11.0, running_avg_reward: -7.54, curr_epsilon: 0.005\n",
      "episode_num 2142, curr_reward: -1.0, best_reward: 11.0, running_avg_reward: -7.42, curr_epsilon: 0.005\n",
      "episode_num 2143, curr_reward: -6.0, best_reward: 11.0, running_avg_reward: -7.43, curr_epsilon: 0.005\n",
      "episode_num 2144, curr_reward: -6.0, best_reward: 11.0, running_avg_reward: -7.38, curr_epsilon: 0.005\n",
      "episode_num 2145, curr_reward: 3.0, best_reward: 11.0, running_avg_reward: -7.21, curr_epsilon: 0.005\n",
      "episode_num 2146, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -7.19, curr_epsilon: 0.005\n",
      "episode_num 2147, curr_reward: -8.0, best_reward: 11.0, running_avg_reward: -7.28, curr_epsilon: 0.005\n",
      "episode_num 2148, curr_reward: -2.0, best_reward: 11.0, running_avg_reward: -7.23, curr_epsilon: 0.005\n",
      "episode_num 2149, curr_reward: -5.0, best_reward: 11.0, running_avg_reward: -7.15, curr_epsilon: 0.005\n",
      "episode_num 2150, curr_reward: -12.0, best_reward: 11.0, running_avg_reward: -7.18, curr_epsilon: 0.005\n",
      "episode_num 2151, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -7.19, curr_epsilon: 0.005\n",
      "episode_num 2152, curr_reward: -5.0, best_reward: 11.0, running_avg_reward: -7.11, curr_epsilon: 0.005\n",
      "episode_num 2153, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -7.09, curr_epsilon: 0.005\n",
      "episode_num 2154, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.1, curr_epsilon: 0.005\n",
      "episode_num 2155, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.11, curr_epsilon: 0.005\n",
      "episode_num 2156, curr_reward: -8.0, best_reward: 11.0, running_avg_reward: -7.18, curr_epsilon: 0.005\n",
      "episode_num 2157, curr_reward: -12.0, best_reward: 11.0, running_avg_reward: -7.24, curr_epsilon: 0.005\n",
      "episode_num 2158, curr_reward: -17.0, best_reward: 11.0, running_avg_reward: -7.38, curr_epsilon: 0.005\n",
      "episode_num 2159, curr_reward: -6.0, best_reward: 11.0, running_avg_reward: -7.32, curr_epsilon: 0.005\n",
      "episode_num 2160, curr_reward: -15.0, best_reward: 11.0, running_avg_reward: -7.43, curr_epsilon: 0.005\n",
      "episode_num 2161, curr_reward: -5.0, best_reward: 11.0, running_avg_reward: -7.44, curr_epsilon: 0.005\n",
      "episode_num 2162, curr_reward: -13.0, best_reward: 11.0, running_avg_reward: -7.46, curr_epsilon: 0.005\n",
      "episode_num 2163, curr_reward: -7.0, best_reward: 11.0, running_avg_reward: -7.52, curr_epsilon: 0.005\n",
      "episode_num 2164, curr_reward: -7.0, best_reward: 11.0, running_avg_reward: -7.56, curr_epsilon: 0.005\n",
      "episode_num 2165, curr_reward: -8.0, best_reward: 11.0, running_avg_reward: -7.58, curr_epsilon: 0.005\n",
      "episode_num 2166, curr_reward: -6.0, best_reward: 11.0, running_avg_reward: -7.53, curr_epsilon: 0.005\n",
      "episode_num 2167, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -7.53, curr_epsilon: 0.005\n",
      "episode_num 2168, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.56, curr_epsilon: 0.005\n",
      "episode_num 2169, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -7.58, curr_epsilon: 0.005\n",
      "episode_num 2170, curr_reward: -4.0, best_reward: 11.0, running_avg_reward: -7.48, curr_epsilon: 0.005\n",
      "episode_num 2171, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.48, curr_epsilon: 0.005\n",
      "episode_num 2172, curr_reward: -15.0, best_reward: 11.0, running_avg_reward: -7.51, curr_epsilon: 0.005\n",
      "episode_num 2173, curr_reward: 1.0, best_reward: 11.0, running_avg_reward: -7.45, curr_epsilon: 0.005\n",
      "episode_num 2174, curr_reward: -16.0, best_reward: 11.0, running_avg_reward: -7.57, curr_epsilon: 0.005\n",
      "episode_num 2175, curr_reward: -3.0, best_reward: 11.0, running_avg_reward: -7.49, curr_epsilon: 0.005\n",
      "episode_num 2176, curr_reward: -12.0, best_reward: 11.0, running_avg_reward: -7.52, curr_epsilon: 0.005\n",
      "episode_num 2177, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.57, curr_epsilon: 0.005\n",
      "episode_num 2178, curr_reward: -14.0, best_reward: 11.0, running_avg_reward: -7.57, curr_epsilon: 0.005\n",
      "episode_num 2179, curr_reward: -13.0, best_reward: 11.0, running_avg_reward: -7.57, curr_epsilon: 0.005\n",
      "episode_num 2180, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -7.56, curr_epsilon: 0.005\n",
      "episode_num 2181, curr_reward: -2.0, best_reward: 11.0, running_avg_reward: -7.52, curr_epsilon: 0.005\n",
      "episode_num 2182, curr_reward: -8.0, best_reward: 11.0, running_avg_reward: -7.56, curr_epsilon: 0.005\n",
      "episode_num 2183, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -7.6, curr_epsilon: 0.005\n",
      "episode_num 2184, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -7.63, curr_epsilon: 0.005\n",
      "episode_num 2185, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -7.65, curr_epsilon: 0.005\n",
      "episode_num 2186, curr_reward: -1.0, best_reward: 11.0, running_avg_reward: -7.57, curr_epsilon: 0.005\n",
      "episode_num 2187, curr_reward: -7.0, best_reward: 11.0, running_avg_reward: -7.55, curr_epsilon: 0.005\n",
      "episode_num 2188, curr_reward: -1.0, best_reward: 11.0, running_avg_reward: -7.44, curr_epsilon: 0.005\n",
      "episode_num 2189, curr_reward: -5.0, best_reward: 11.0, running_avg_reward: -7.38, curr_epsilon: 0.005\n",
      "episode_num 2190, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -7.41, curr_epsilon: 0.005\n",
      "episode_num 2191, curr_reward: -5.0, best_reward: 11.0, running_avg_reward: -7.41, curr_epsilon: 0.005\n",
      "episode_num 2192, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -7.49, curr_epsilon: 0.005\n",
      "episode_num 2193, curr_reward: -1.0, best_reward: 11.0, running_avg_reward: -7.34, curr_epsilon: 0.005\n",
      "episode_num 2194, curr_reward: -3.0, best_reward: 11.0, running_avg_reward: -7.31, curr_epsilon: 0.005\n",
      "episode_num 2195, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.25, curr_epsilon: 0.005\n",
      "episode_num 2196, curr_reward: -11.0, best_reward: 11.0, running_avg_reward: -7.29, curr_epsilon: 0.005\n",
      "episode_num 2197, curr_reward: -8.0, best_reward: 11.0, running_avg_reward: -7.25, curr_epsilon: 0.005\n",
      "episode_num 2198, curr_reward: -3.0, best_reward: 11.0, running_avg_reward: -7.27, curr_epsilon: 0.005\n",
      "episode_num 2199, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.4, curr_epsilon: 0.005\n",
      "episode_num 2200, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.43, curr_epsilon: 0.005\n",
      "episode_num 2201, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.51, curr_epsilon: 0.005\n",
      "episode_num 2202, curr_reward: -15.0, best_reward: 11.0, running_avg_reward: -7.58, curr_epsilon: 0.005\n",
      "episode_num 2203, curr_reward: -15.0, best_reward: 11.0, running_avg_reward: -7.65, curr_epsilon: 0.005\n",
      "episode_num 2204, curr_reward: -15.0, best_reward: 11.0, running_avg_reward: -7.83, curr_epsilon: 0.005\n",
      "episode_num 2205, curr_reward: -9.0, best_reward: 11.0, running_avg_reward: -7.78, curr_epsilon: 0.005\n",
      "episode_num 2206, curr_reward: -7.0, best_reward: 11.0, running_avg_reward: -7.74, curr_epsilon: 0.005\n",
      "episode_num 2207, curr_reward: -7.0, best_reward: 11.0, running_avg_reward: -7.75, curr_epsilon: 0.005\n",
      "episode_num 2208, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.74, curr_epsilon: 0.005\n",
      "episode_num 2209, curr_reward: -8.0, best_reward: 11.0, running_avg_reward: -7.92, curr_epsilon: 0.005\n",
      "episode_num 2210, curr_reward: -4.0, best_reward: 11.0, running_avg_reward: -7.86, curr_epsilon: 0.005\n",
      "episode_num 2211, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.91, curr_epsilon: 0.005\n",
      "episode_num 2212, curr_reward: -7.0, best_reward: 11.0, running_avg_reward: -7.91, curr_epsilon: 0.005\n",
      "episode_num 2213, curr_reward: -6.0, best_reward: 11.0, running_avg_reward: -7.91, curr_epsilon: 0.005\n",
      "episode_num 2214, curr_reward: -14.0, best_reward: 11.0, running_avg_reward: -7.91, curr_epsilon: 0.005\n",
      "episode_num 2215, curr_reward: -7.0, best_reward: 11.0, running_avg_reward: -7.96, curr_epsilon: 0.005\n",
      "episode_num 2216, curr_reward: -10.0, best_reward: 11.0, running_avg_reward: -7.95, curr_epsilon: 0.005\n",
      "episode_num 2217, curr_reward: -13.0, best_reward: 11.0, running_avg_reward: -8.03, curr_epsilon: 0.005\n",
      "episode_num 2218, curr_reward: -2.0, best_reward: 11.0, running_avg_reward: -7.98, curr_epsilon: 0.005\n"
     ]
    }
   ],
   "source": [
    "env = wrap_env(ENV)\n",
    "dvc = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "mdl, tgt_mdl = models_init(env, dvc)\n",
    "opt = Adam(mdl.parameters(), lr=LR)\n",
    "rpl_bfr = RepBfr(MEM_CAP)\n",
    "train(env, mdl, tgt_mdl, opt, rpl_bfr, dvc)\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
